Commit 76415a20 authored by Quentin David's avatar Quentin David
Browse files

Renaming some stuff, began to implementing in detail the API, adding stuff for login

parent 61943d3f
const env = process.env.NODE_ENV || 'development';
const config = {
production: {
secret: process.env.JWT_SECRET || '',
ldap: {
url: process.env.LDAP_URL || '',
bindDN: process.env.LDAP_ADMIN_USER || '',
bindCredentials: process.env.LDAP_ADMIN_PWD || '',
searchBase: process.env.LDAP_BASE || 'ou=people,dc=univ-paris13,dc=fr',
searchFilter: process.env.LDAP_FILTER || '(&(uid={{username}})(accountStatus=active*))',
searchAttributes: process.env.LDAP_ATTRIBUTES || 'uid, mail, displayName, accountStatus'
}
},
development: {
secret: process.env.DEV_JWT_SECRET || 'lipn-cat',
ldap: {
url: process.env.DEV_LDAP_URL || 'ldap://0.0.0.0:389',
searchBase: process.env.DEV_LDAP_BASE || 'ou=people,dc=planetexpress,dc=com',
searchFilter: process.env.DEV_LDAP_FILTER || '(uid={{username}})',
searchAttributes: process.env.DEV_LDAP_ATTRIBUTES || 'uid, mail, cn',
passwordField: 'userPassword'
}
},
test: {
secret: process.env.TEST_JWT_SECRET || 'lipn-cat'
}
};
module.exports = config[env];
const LDAPStrategy = require('passport-ldapauth');
const JWTStrategy = require('passport-jwt').Strategy;
const ExtractJwt = require('passport-jwt').ExtractJwt;
const { secret, ldap } = require('./auth.js');
module.exports = function (passport) {
// JWT Configuration
const jwtOptions = {
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
secretOrKey: secret
};
// Use passport with JWT strategy
passport.use(new JWTStrategy(jwtOptions,
function (payload, done) {
console.log('payload received', payload);
done(null, payload);
}));
// Use passport with the LDAP strategy
const searchAttributes = ldap.searchAttributes.split(',').map(e => e.trim());
const ldapOpts = { server: { ...ldap, searchAttributes } };
passport.use(new LDAPStrategy(ldapOpts));
};
...@@ -20,14 +20,28 @@ exports.debug_corpus_list = function(req, res) { ...@@ -20,14 +20,28 @@ exports.debug_corpus_list = function(req, res) {
} }
// Add a new corpus // Add a new corpus
exports.corpus_create = function(req,res) { exports.corpus_create = function(req,res) {
console.log("Demande d'ajout d'un corpus:")
console.log(req.body);
const newCorpus = CorpusModel({ const newCorpus = CorpusModel({
corpus_name: req.body.corpus_name, // documents: ....
content: req.body.content, createdBy: req.body.createdBy,
created_by: req.body.created_by type: req.body.type,
creationDate: req.body.creationDate,
metadata: {
author: req.body.metadata.author,
title: req.body.metadata.title,
description: req.body.metadata.description,
date: req.body.metadata.date,
type: req.body.metadata.type,
size: req.body.metadata.size,
language: req.body.metadata.language,
additionalMetadata: req.body.metadata.additionalMetadata,
}
}); });
newCorpus.save((error, corpus) => { newCorpus.save((error, corpus) => {
if (error) { errorHandler(res, error); } if (error) { errorHandler(res, error); }
else { else {
console.log("Successfully added the corpus")
successHandler(res, { corpus: corpus}, 201); successHandler(res, { corpus: corpus}, 201);
} }
}) })
...@@ -80,7 +94,16 @@ exports.corpus_summary = function(req,res) { ...@@ -80,7 +94,16 @@ exports.corpus_summary = function(req,res) {
failHandler(res, { corpus: `No corpus was found with id: ${corpusId}`}, 404) failHandler(res, { corpus: `No corpus was found with id: ${corpusId}`}, 404)
} }
else { else {
successHandler(res, { corpus_summary: 'TBD' }); // Almost the same than the model, except we only get the number of docs, and not its content.
const corpusSummary = {
corpusId: corpusId,
createdBy: corpus.createdBy,
creationDate: corpus.creationDate,
type: corpus.type,
documentNumber: corpus.documents.length,
metadata: corpus.metadata
};
successHandler(res, { corpusSummary: corpusSummary });
} }
}) })
} }
......
...@@ -69,6 +69,20 @@ exports.corpus_delete = function(req,res) { ...@@ -69,6 +69,20 @@ exports.corpus_delete = function(req,res) {
} }
}); });
} }
// Get the conllu of an existing corpus process
exports.corpus_process_conllu_get = function(res, req) {
const corpusProcessId = req.params.corpusProcesId;
CorpusProcessModel.findById(corpusProcessId, (error, corpusProcess) => {
if(error) { errorHandler(res, error); }
if (!corpusProcess) {
failHandler(res, { corpusProcess: `No corpus process was found with id: ${corpusProcessId}`}, 404);
}
else {
// TO BE FILLED
successHandler(res, { corpusProcess: { conllu: 'TBD'} });
}
})
}
// Get the status of an existing corpus process // Get the status of an existing corpus process
exports.corpus_process_status_get = function(req,res) { exports.corpus_process_status_get = function(req,res) {
const corpusProcessId = req.params.corpusProcessId; const corpusProcessId = req.params.corpusProcessId;
......
const CorpusModel = require('../../models/Corpus.js');
const { errorHandler, failHandler } = require('./responseHelper.js');
// TEST: Unknown id -> it should be 404 and no goes to successhandler of userController (corpora list)
function getSummaryFromId(res, corpusId) {
console.log('Corpus Id:');
console.log(corpusId);
CorpusModel.findById(corpusId, (error, corpus) => {
if (error) { errorHandler(res, error); }
if (!corpus) {
failHandler(res, { corpus: `No corpus was found with id: ${corpusId}`}, 404)
}
else {
console.log('Found the corpus');
// Almost the same than the model, except we only get the number of docs, and not its content.
const corpusSummary = {
corpusId: corpusId,
createdBy: corpus.createdBy,
creationDate: corpus.creationDate,
type: corpus.type,
documentNumber: corpus.documents.length,
metadata: corpus.metadata
};
console.log("Corpus Summary in its function");
console.log(corpusSummary);
return corpusSummary;
// successHandler(res, { corpusSummary: corpusSummary });
}
})
}
// Need number of documents
exports.getSummaryListFromIds = function (res, corpusIdList) {
CorpusModel.find( {'_id': {$in: corpusIdList}}, 'createdBy creationDate type metadata', (error, corpusSummaryList) => {
if (error) {
console.log(error);
errorHandler(res, error); }
// No corpora were found at all
if (!corpusSummaryList) {
// to be filled
//failHandler(res )
pass
}
// Some of the corpora were not found
if (corpusSummaryList.length < corpusIdList.length) {
// To be filled
pass
}
else {
console.log("Found corpusSummary list");
console.log(corpusSummaryList);
return corpusSummaryList;
}
})
}
exports.getSummaryListFromIds2 = function (res, corpusIdList) {
var corpusSummaryList = [];
var corpusSummary;
corpusIdList.forEach((element) => {
corpusSummary = getSummaryFromId(res, element);
console.log("corpus summary:");
console.log(corpusSummary)
corpusSummaryList.push(corpusSummary);
});
console.log("corpussummarylist");
console.log(corpusSummaryList)
return corpusSummaryList;
}
\ No newline at end of file
exports.errorHandler = function(res, error) { exports.errorHandler = function(res, error, httpCode = 500) {
// I'm not sure howExpress deals with http code and error, does it always send200 OK ? res.status(httpCode);
res.json({ res.json({
status: "error", status: "error",
message: error message: error
......
...@@ -9,6 +9,8 @@ const errorHandler = responseHelper.errorHandler; ...@@ -9,6 +9,8 @@ const errorHandler = responseHelper.errorHandler;
const successHandler = responseHelper.successHandler; const successHandler = responseHelper.successHandler;
const failHandler = responseHelper.failHandler; // For eg. "User not found" is a fail, not an error const failHandler = responseHelper.failHandler; // For eg. "User not found" is a fail, not an error
const corpusHelper = require('./lib/corpusHelper');
/* User personal information management functions */ /* User personal information management functions */
// DEBUG: User list // DEBUG: User list
...@@ -81,17 +83,22 @@ exports.user_delete = function(req,res) { ...@@ -81,17 +83,22 @@ exports.user_delete = function(req,res) {
exports.user_corpus_list = function(req,res) { exports.user_corpus_list = function(req,res) {
// TODO: Fetch the corpora from the corpus DB // TODO: Fetch the corpora from the corpus DB
const userId = req.params.userId; const userId = req.params.userId;
UserModel.findById(userId, (error, user) => { UserModel.findById(userId, async (error, user) => {
if (error) { errorHandler(res, error); } if (error) { errorHandler(res, error); }
else if (!user) { else if (!user) {
failHandler(res, { user: `No user was found with id: ${userId}`}, 404); failHandler(res, { user: `No user was found with id: ${userId}`}, 404);
} }
else { else {
// Should fetch /corpus to get the summary ? // ProfileCorpora is a list of ids.
successHandler(res, { profileCorpora: user.profileCorpora}); // Promise ici non ?
var profileCorporaList = await corpusHelper.getSummaryListFromIds(res, user.profileCorpora)
console.log("Juste before success handler");
console.log(typeof profileCorporaList);
successHandler(res, { profileCorpora: profileCorporaList });
} }
}) })
} }
// Add a new corpus to an existing user // Add a new corpus to an existing user
exports.user_corpus_create = function(req,res) { exports.user_corpus_create = function(req,res) {
const userId = req.params.userId; const userId = req.params.userId;
...@@ -102,9 +109,10 @@ exports.user_corpus_create = function(req,res) { ...@@ -102,9 +109,10 @@ exports.user_corpus_create = function(req,res) {
} }
else { else {
const corpusId = req.body.corpusId; const corpusId = req.body.corpusId;
const newCorpus = { corpusId: corpusId};
user.profileCorpora.push(newCorpus); // const newCorpus = { corpusId: corpusId};
user.profileCorpora.push(corpusId);
user.save((error, _) => { user.save((error, _) => {
if (error) { errorHandler(res, error); } if (error) { errorHandler(res, error); }
else { else {
......
...@@ -46,10 +46,18 @@ Request Body: ...@@ -46,10 +46,18 @@ Request Body:
``` ```
System will now maybe check all the informations, then try to fill conlluBuffer of each documents by tokenizing the corpus. It will also need to compute the size of the corpus. System will now maybe check all the informations, then try to fill conlluBuffer of each documents by tokenizing the corpus. It will also need to compute the size of the corpus.
Informations intéressantes:
- Nombre de mots / tokens
- Taille informatique (du fichier source)
From now on, this document (as in a mongo document) will bear the id "corpusId". From now on, this document (as in a mongo document) will bear the id "corpusId".
## II: User set their pipeline ## II: User set their pipeline
PRETREATMENTS:
La segmentation par phrases et par mots se fera systématiquement lors du conllu buffer, on peut donc enlever ces propriétés de preTreatments.
POST /pipelines POST /pipelines
Request body: Request body:
...@@ -101,32 +109,32 @@ Request body: ...@@ -101,32 +109,32 @@ Request body:
corpusId: corpusId, corpusId: corpusId,
pipelineId: pipelineId, pipelineId: pipelineId,
userId: 'myUserId', userId: 'myUserId',
conllu: null, // Empty until filled by modules conllu: null, // Will be computed after the POST
annotatedDocuments: [ visualAnnotatedDocuments: [], // It will be computed after the pipeline
{
documentId: article1DocumentId,
corpusProcessId: null, // Same issue than in I: id not yet created
annotations: []
}
],
outputs: null, outputs: null,
currentProcessModule: null, currentProcessModule: null,
status: 'Not started yet' status: 'Not started yet'
} }
``` ```
On concatène tous les conlluBuffer de chaque document du corpus «corpusId» (GET /corpus/corpusId/...)
PUT /corpusProcesses/corpusProcessId
pour ajouter le conllu concaténer.
### Pre-Treatments executions ### Pre-Treatments executions
Once it is added to the collection, it will fetch the preTreatments in the document of the corresponding pipeline. Once it is added to the collection, it will fetch the preTreatments in the document of the corresponding pipeline.
``` javascript ``` javascript
preTreatments: { preTreatments: {
sentenceSegmentation: false, //sentenceSegmentation: false,
wordSegmentation: true, //wordSegmentation: true,
posTagger: true, posTagger: true,
conversionToUTF: false codeConversionToUTF: false
} }
``` ```
wordSegmentation is supposed to be already done as we posted the corpus. wordSegmentation is supposed to be already done as we posted the corpus.
So the system will need to apply the POS Tagger to the corpus. So the system will need to apply the POS Tagger to the corpus.
...@@ -143,7 +151,7 @@ Then apply the process, then returning a conllu column that we can add to corpus ...@@ -143,7 +151,7 @@ Then apply the process, then returning a conllu column that we can add to corpus
Once it is added to the collection, it will fetch the first process in the list of the corresponding pipelineId and call the module, then update: Once it is added to the collection, it will fetch the first process in the list of the corresponding pipelineId and call the module, then update:
POST /modules/Neoveille > POST /modules/Neoveille
Request body: Request body:
``` javascript ``` javascript
...@@ -165,17 +173,16 @@ Request body: ...@@ -165,17 +173,16 @@ Request body:
Request body: Request body:
``` javascript ``` javascript
{ {
currentProcessingModule: 'Neoveille', currentProcessingModule: 'Neoveille', // null -> Neoveille
status: 'Started' status: 'Started' // Not started yet -> Started
} }
``` ```
## IV: Module Processing ## IV: Module Processing
The module will need to fetch the conllu. The module will need to fetch the conllu.
(need a specific GET for conllu?)
> GET /corpusProcess/corpusProcessId > GET /corpusProcesses/corpusProcessId/conllu
Then, given parameters given from the previous paragraph, applies the process to the conllu. Then, given parameters given from the previous paragraph, applies the process to the conllu.
...@@ -183,7 +190,7 @@ Neoveille will then give back two things: ...@@ -183,7 +190,7 @@ Neoveille will then give back two things:
- one output: the raw list of found neologisms - one output: the raw list of found neologisms
- one conllu column with - one conllu column with
> POST /corpusProcess/corpusProcessId > POST /corpusProcess/corpusProcessId/processOutputs
Request body: Request body:
``` javascript ``` javascript
...@@ -209,6 +216,8 @@ This request adds those two outputs of neoveille process to the collection corpu ...@@ -209,6 +216,8 @@ This request adds those two outputs of neoveille process to the collection corpu
### When Neoveille is finised... ### When Neoveille is finised...
(Comment le système sait quand Neoveille finit ?)
We can start the next module on the pipeline list. We can start the next module on the pipeline list.
> POST /modules/SDMC > POST /modules/SDMC
...@@ -227,7 +236,7 @@ Request body: ...@@ -227,7 +236,7 @@ Request body:
``` ```
Then update our corpusProcess Then update our corpusProcess
> PUT /corpusProcess/corpusProcessId > PUT /corpusProcesses/corpusProcessId
Request body: Request body:
``` javascript ``` javascript
...@@ -238,11 +247,11 @@ Request body: ...@@ -238,11 +247,11 @@ Request body:
Now, similar to Neoveille, SDMC will need to fetch the conllu, Now, similar to Neoveille, SDMC will need to fetch the conllu,
> GET /corpusProcess/corpusProcessId > GET /corpusProcesses/corpusProcessId/conllu
Process that corpus, then adds its output to the structure Process that corpus, then adds its output to the structure
> POST /corpusProcess/corpusProcessId > POST /corpusProcesses/corpusProcessId/processOutputs
Request body: Request body:
``` javascript ``` javascript
...@@ -276,12 +285,15 @@ First, the system extrates the conllu. ...@@ -276,12 +285,15 @@ First, the system extrates the conllu.
Then it processes it and adds to annotated documents the annotations: Then it processes it and adds to annotated documents the annotations:
> PUT /corpusProcesses/corpusProcessId (/annotation?) > POST /corpusProcesses/corpusProcessId/visualAnnotatedDocument
On coupe le conllu pour chaque document.
On traite pour chaque document toutes ses colonnes
Request body: Request body:
``` javascript ``` javascript
{ {
annotatedDocuments:[ newVisualAnotatedDocument:[
{ {
documentId: 'article1DocumentId', documentId: 'article1DocumentId',
annotations: [ annotations: [
...@@ -290,6 +302,18 @@ Request body: ...@@ -290,6 +302,18 @@ Request body:
processId: 'NeoveilleProcess', processId: 'NeoveilleProcess',
conlluColumnId: 'conlluNeoveilleId', conlluColumnId: 'conlluNeoveilleId',
moduleName: 'Neoveille', moduleName: 'Neoveille',
content: {
title: 'Annotation des Néologismes',
description: 'Trouvés par Néoveille selon ces paramètres:',
data: null // ??
},
color: null // computed later ?
},
{
documentId: 'article1DocumentId',
processId: 'SDMCProcess',
conlluColumnId: 'conlluSDMCId',
moduleName: 'Neoveille',
content: { content: {
title: 'Annotation des Néologismes', title: 'Annotation des Néologismes',
description: 'Trouvés par Néoveille selon ces paramètres:', description: 'Trouvés par Néoveille selon ces paramètres:',
...@@ -299,4 +323,24 @@ Request body: ...@@ -299,4 +323,24 @@ Request body:
] ]
} }
] ]
} }
\ No newline at end of file ```
Il faudrait faire un calcul pour affecter les couleurs pour chaque annotation des documents.
Tout le process est terminé, on peut mettre à jour lestatus de corpusProcess
> PUT /corpusProcesses/corpusProcessId
Request body:
``` javascript
{
status: 'Finished'
}
```
Et on envoie un mail à l'utilisateur s'il a activé l'option.
( POST /users/myUserId/sendMail )
...@@ -73,7 +73,7 @@ const DocumentSchema = new Schema({ ...@@ -73,7 +73,7 @@ const DocumentSchema = new Schema({
* type: String * type: String
* description: language of the corpus, can be multilingual * description: language of the corpus, can be multilingual
* example: fr * example: fr
* userMetadata: * additionalMetadata:
* type: array * type: array
* description: list of metadata added by the user * description: list of metadata added by the user
* items: * items:
...@@ -98,7 +98,7 @@ const MetadataSchema = new Schema({ ...@@ -98,7 +98,7 @@ const MetadataSchema = new Schema({
type: {type: String }, type: {type: String },
size: {type: String }, size: {type: String },
language: {type: String }, language: {type: String },
userMetadata: [{ additionalMetadata: [{
name: {type: String }, name: {type: String },
value: {type: String } value: {type: String }
}] }]
...@@ -145,6 +145,15 @@ const CorpusSchema = new Schema({ ...@@ -145,6 +145,15 @@ const CorpusSchema = new Schema({
metadata: { type: MetadataSchema } metadata: { type: MetadataSchema }
}); });
CorpusSchema.virtual('numberOfDocuments').get(function() {
if (this.documents) {
return this.documents.length;
}
else {
return 0;
}
});
// Let's compile as a model // Let's compile as a model
const CorpusModel = mongoose.model('CorpusModel', CorpusSchema); const CorpusModel = mongoose.model('CorpusModel', CorpusSchema);
......
...@@ -226,6 +226,7 @@ const AnnotatedDocumentSchema = new Schema({ ...@@ -226,6 +226,7 @@ const AnnotatedDocumentSchema = new Schema({
* - Not started yet * - Not started yet
* - Started * - Started
* - Finished