Commit a9eed2fe authored by Petter Goksøyr Åsen's avatar Petter Goksøyr Åsen
Browse files

DEICH-5603 fetch work data from euler, and not directly from virtuoso

parent 099fc303
const routes = require("express").Router();
const fetch = require("isomorphic-unfetch");
const uniqBy = require("lodash/uniqBy");
const sparqlQueries = require("../utils/sparqlQueries");
const {
parseRDFtoJsonLD,
frameJsonLD,
addItemsToPublications
} = require("../utils/resourceHelpers");
......@@ -14,7 +12,6 @@ const logger = require("../../logger")(__filename);
const CALL_ID_HEADER = "Deichman-CallID";
const TIMEOUT_MS = 5000;
const kohaEndpoint = process.env.INTERNAL_URL_KOHA;
const virtuosoEndpoint = process.env.SPARQL_ENDPOINT;
const { INTERNAL_URL_EULER, INTERNAL_URL_SIBYL } = process.env;
// ----- Shared promises -----
......@@ -24,21 +21,21 @@ const publicationLinkScore = binding => {
if (
binding.mediaType &&
binding.targetMediaType &&
binding.mediaType.value === binding.targetMediaType.value
binding.mediaType === binding.targetMediaType
) {
score++;
}
if (
binding.format &&
binding.targetFormat &&
binding.format.value === binding.targetFormat.value
binding.format === binding.targetFormat
) {
score++;
}
if (
binding.lang &&
binding.targetLang &&
binding.lang.value === binding.targetLang.value
binding.lang === binding.targetLang
) {
score += 3;
}
......@@ -46,13 +43,9 @@ const publicationLinkScore = binding => {
};
async function decorateRelation(key, publicationId, relation, deichmanCallId) {
const query = sparqlQueries.decorateRelation(publicationId, relation.uri);
return fetch(virtuosoEndpoint, {
method: "POST",
body: `query=${encodeURIComponent(query)}`,
const url = `${INTERNAL_URL_EULER}/api/authorities/decorateRelation?id=${publicationId}&relation=${encodeURIComponent(relation.uri)}`;
return fetch(url, {
headers: {
Accept: "application/sparql-results+json",
"Content-Type": "application/x-www-form-urlencoded",
"Deichman-CallID": deichmanCallId
}
})
......@@ -64,26 +57,26 @@ async function decorateRelation(key, publicationId, relation, deichmanCallId) {
})
.then(json => {
let selected = false;
json.results.bindings.forEach(binding => {
(json || []).forEach(binding => {
const score = publicationLinkScore(binding);
if (!selected || score > selected.score) {
if (binding.targetPub) {
selected = {
score,
publicationId: binding.targetPub.value.replace(
publicationId: binding.targetPub.replace(
"http://data.deichman.no/publication/",
""
),
mainTitle: binding.targetMainTitle.value
mainTitle: binding.targetMainTitle
};
if (binding.targetSubtitle) {
selected.subtitle = binding.targetSubtitle.value;
selected.subtitle = binding.targetSubtitle;
}
if (binding.targetPartNumber) {
selected.partNumber = binding.targetPartNumber.value;
selected.partNumber = binding.targetPartNumber;
}
if (binding.targetPartTitle) {
selected.partTitle = binding.targetPartTitle.value;
selected.partTitle = binding.targetPartTitle;
}
}
}
......@@ -95,11 +88,11 @@ async function decorateRelation(key, publicationId, relation, deichmanCallId) {
relation.partTitle = selected.partTitle;
if (
!selected &&
json.results.bindings.length > 0 &&
json.results.bindings[0].workMainTitle
json.length > 0 &&
json[0].workMainTitle
) {
// Related work has no publications yet, so we use the title from the work.
relation.mainTitle = json.results.bindings[0].workMainTitle.value;
relation.mainTitle = json[0].workMainTitle;
}
return { key, relation };
});
......@@ -151,69 +144,36 @@ function addRecommendationsAndLikesToPublication(
}
const getWorkDataByPublication = (publicationId, deichmanCallId) => {
const query = sparqlQueries.describeWorkViaPublication(publicationId);
return fetch(virtuosoEndpoint, {
method: "POST",
body: `query=${encodeURIComponent(query)}`,
headers: {
Accept: "application/n-triples;charset=utf-8",
"Content-Type": "application/x-www-form-urlencoded",
"Deichman-CallID": deichmanCallId
}
})
.then(res => {
if (res.status === 200) {
return res.text();
}
const url = `${INTERNAL_URL_EULER}/api/authorities/expandedWorkByPublication/${publicationId}`;
return fetch(url, {
method: "GET",
headers: { "Deichman-CallID": deichmanCallId }
}).then(res => {
if (res.status !== 200) {
return Promise.reject(
new Error(`error fetching work ${publicationId}: ${res.statusText}`)
);
})
.then(ntdata => {
if (!ntdata || ntdata.length < 100) {
// virtuoso returns one line with comment: # Empty NT
}
return res.json()
.then(jsonld => {
if (jsonld.length === 0 ) { // empty json-ld
return Promise.reject(
new Error(`error fetching work ${publicationId}: 404 not found`)
new Error(`404 publication ${publicationId} not found}`)
);
}
return parseRDFtoJsonLD(ntdata, publicationId);
return frameJsonLD(jsonld, publicationId)
})
.then(ntdoc => frameJsonLD(ntdoc));
})
};
// TODO rethink this function - we allready have the recordIds in the response from getWorkDataByPublication
const getWorkItemsByPublication = (
publicationId,
categorycode,
deichmanCallId
const getWorkItemsByRecordIds = (
recordIds,
categorycode
) => {
const query = sparqlQueries.recordIdsByWorkViaPublication(publicationId);
return fetch(virtuosoEndpoint, {
method: "POST",
body: `query=${encodeURIComponent(query)}`,
headers: {
Accept: "application/sparql-results+json",
"Content-Type": "application/x-www-form-urlencoded",
"Deichman-CallID": deichmanCallId
}
})
.then(res => {
if (res.status === 200) {
return res.json();
}
logger.info(
`ERROR fetching items for work by publication ${publicationId}: ${
res.statusText
}`,
{ publication_id: publicationId, call_id: deichmanCallId }
);
return Promise.resolve({ results: { bindings: [] } }); // We don't wan't to fail if items cannot be found
})
.then(json =>
Promise.all(
json.results.bindings.map(binding =>
return Promise.all(
recordIds.map(id =>
fetch(
`${kohaEndpoint}/api/biblio/${binding.recordId.value}/expanded${
`${kohaEndpoint}/api/biblio/${id}/expanded${
categorycode ? "?categorycode=" + categorycode : ""
}`
).then(res => {
......@@ -223,81 +183,80 @@ const getWorkItemsByPublication = (
return Promise.resolve(undefined);
})
)
).then(itemResponses => {
const itemsByRecordId = {};
const holdsByRecordId = {};
itemResponses
.filter(itemResponse => itemResponse)
.forEach(itemResponse => {
holdsByRecordId[itemResponse.biblio.biblionumber] =
itemResponse.biblio;
const items = {};
itemResponse.items.forEach(item => {
if (item.status === "Utilgjengelig") {
// Theese items should not count, or be processed further,
// as they are not available to end users.
return;
}
const newItem = {
shelfmark: item.itemcallnumber,
status: item.status,
reservable: item.reservable === 1,
branchcode: item.homebranch,
barcode: item.barcode,
notforloan:
item.status === "Ikke til hjemlån" ||
item.status === "Ikke til hjemlån (utlånt)",
notforloanAndCheckedOut:
item.status === "Ikke til hjemlån (utlånt)"
};
if (item.newLocation) {
const { livesLabel = "", coLabel = "" } = item.newLocation;
newItem.locLabel = coLabel ? coLabel : livesLabel;
if (
item.newLocation.meta.poi &&
item.newLocation.meta.poi.url
) {
newItem.poi = item.newLocation.meta.poi.url;
}
}
const key = `${newItem.branchcode}_${newItem.shelfmark}_${
newItem.locLabel
}_${newItem.notforloan}`;
).then(itemResponses => {
const itemsByRecordId = {};
const holdsByRecordId = {};
itemResponses
.filter(itemResponse => itemResponse)
.forEach(itemResponse => {
  • hadde det gitt mening å flytte mer av dette ut til Euler, eller er det presentasjonslogikk?

  • Det er jo noe vi har snakket om, men i dag så går ingenting fra koha via euler, så vi bør vel ha en grundig diskusjon og plan først. Kanskje det gir mer mening å massere ferdig i koha sine apier

Please register or sign in to reply
holdsByRecordId[itemResponse.biblio.biblionumber] =
itemResponse.biblio;
const items = {};
itemResponse.items.forEach(item => {
if (item.status === "Utilgjengelig") {
  • Hvor kommer disse strengene fra? Har vi ikke en ID eller noe flagg vi kan bruke? Dette er kode som på magisk og umerkelig vis plutselig slutter å fungerer etter min erfaring.

  • De kommer fra koha. Denne delen av koden har ikke blitt endret i denne omgangen, kun flyttet litt rundt på.

Please register or sign in to reply
// Theese items should not count, or be processed further,
// as they are not available to end users.
return;
}
const newItem = {
shelfmark: item.itemcallnumber,
status: item.status,
reservable: item.reservable === 1,
branchcode: item.homebranch,
barcode: item.barcode,
notforloan:
item.status === "Ikke til hjemlån" ||
item.status === "Ikke til hjemlån (utlånt)",
notforloanAndCheckedOut:
item.status === "Ikke til hjemlån (utlånt)"
};
if (!items[key]) {
newItem.available = 0;
newItem.total = 0;
items[key] = newItem;
}
if (item.newLocation) {
const { livesLabel = "", coLabel = "" } = item.newLocation;
newItem.locLabel = coLabel ? coLabel : livesLabel;
if (
newItem.status === "Ledig" ||
(newItem.notforloan && !newItem.notforloanAndCheckedOut)
item.newLocation.meta.poi &&
  • LItt magisk... Kan du legge til en kommentar på hva denne sjekken gjør?

  • Dette er gammel kode, kun whitespace endringer som ser litt rare ut her tror jeg. Har ikke endret på dette

  • Vet du hva koden gjør?

  • Infløkte regler for hva som bestemmer om et eksemplar er tilgjengelig eller ei tror jeg? Burde vel skjedd på et lavere lag (koha). Det ser litt rarere ut i denne diffen en i filen slik den er. Det virker som jeg har endra på mye greier men det er bare indenteringen som er endra

Please register or sign in to reply
item.newLocation.meta.poi.url
) {
items[key].available++;
}
items[key].total++;
if (newItem.reservable) {
items[key].reservable = true;
newItem.poi = item.newLocation.meta.poi.url;
}
});
itemsByRecordId[itemResponse.biblio.biblionumber] = items;
}
const key = `${newItem.branchcode}_${newItem.shelfmark}_${
newItem.locLabel
}_${newItem.notforloan}`;
if (!items[key]) {
newItem.available = 0;
newItem.total = 0;
items[key] = newItem;
}
if (
newItem.status === "Ledig" ||
(newItem.notforloan && !newItem.notforloanAndCheckedOut)
) {
items[key].available++;
}
items[key].total++;
if (newItem.reservable) {
items[key].reservable = true;
}
});
const publications = {};
Object.keys(itemsByRecordId).forEach(key => {
publications[key] = {
items: Object.keys(itemsByRecordId[key]).map(
key2 => itemsByRecordId[key][key2]
)
};
});
Object.keys(holdsByRecordId).forEach(recordId => {
publications[recordId].numHolds = holdsByRecordId[recordId].numholds;
itemsByRecordId[itemResponse.biblio.biblionumber] = items;
});
return publications;
})
);
const publications = {};
Object.keys(itemsByRecordId).forEach(key => {
publications[key] = {
items: Object.keys(itemsByRecordId[key]).map(
key2 => itemsByRecordId[key][key2]
)
};
});
Object.keys(holdsByRecordId).forEach(recordId => {
publications[recordId].numHolds = holdsByRecordId[recordId].numholds;
});
return publications;
})
};
// Get objects referring to a list of publications, if they exist
......@@ -503,16 +462,11 @@ routes.get("/publication/:publicationId/detailed", async (req, res, next) => {
call_id: deichmanCallId
});
try {
const results = await Promise.all([
getWorkDataByPublication(req.params.publicationId, deichmanCallId),
getWorkItemsByPublication(
req.params.publicationId,
req.query.categorycode,
deichmanCallId
)
]);
const workWithoutLinks = await getWorkDataByPublication(req.params.publicationId, deichmanCallId);
const recordIds = workWithoutLinks.publications.map(p => p.recordId);
const items = await getWorkItemsByRecordIds(recordIds, req.query.categorycode);
const work = await addWorkLinks(
results[0],
workWithoutLinks,
req.params.publicationId,
deichmanCallId
);
......@@ -557,7 +511,7 @@ routes.get("/publication/:publicationId/detailed", async (req, res, next) => {
return true;
});
const likes = recommendationsAndLikesResults[2];
const publication = addItemsToPublications(work, results[1]);
const publication = addItemsToPublications(work, items);
const publicationWithRecommendationsAndLikes = addRecommendationsAndLikesToPublication(
publication,
filteredRecommendations,
......
......@@ -265,54 +265,44 @@ function addItemsToPublications(work, items) {
};
}
function parseRDFtoJsonLD(ntdata, publicationId) {
return new Promise((resolve, reject) => {
jsonld.fromRDF(ntdata, { format: "application/nquads" }, (error, ntdoc) => {
if (error) {
reject(error);
}
function transformJsonLD(data, publicationId) {
// First we need to find the workId
let workId;
for (let i = 0; i < data.length; i++) {
const el = data[i];
if (
el["@id"] ===
`http://data.deichman.no/publication/${publicationId}` &&
el["http://data.deichman.no/ontology#publicationOf"] !== ""
) {
workId =
el["http://data.deichman.no/ontology#publicationOf"][0]["@id"];
break;
// TODO handle bad data we have some publications with multiple works..
}
}
// First we need to find the workId
let workId;
for (let i = 0; i < ntdoc.length; i++) {
const el = ntdoc[i];
if (
el["@id"] ===
`http://data.deichman.no/publication/${publicationId}` &&
el["http://data.deichman.no/ontology#publicationOf"] !== ""
) {
workId =
el["http://data.deichman.no/ontology#publicationOf"][0]["@id"];
break;
// TODO handle bad data we have some publications with multiple works..
}
// We need to add a class to the work 'in focus', in case there are other works
// in the graph (for example work as subject of work), and use this class
// for framing.
//
// We also delete the class migration:Work, as it trips up the framing.
return data.map(el => {
if (
el["@type"] &&
el["@type"].includes("http://data.deichman.no/ontology#Work") &&
el["@id"] === workId
) {
el["@type"].push("http://data.deichman.no/ontology#WorkInFocus");
}
if (el["@type"]) {
const i = el["@type"].indexOf("http://migration.deichman.no/Work");
if (i !== -1) {
el["@type"].splice(i, 1);
}
}
// We need to add a class to the work 'in focus', in case there are other works
// in the graph (for example work as subject of work), and use this class
// for framing.
//
// We also delete the class migration:Work, as it trips up the framing.
ntdoc = ntdoc.map(el => {
if (
el["@type"] &&
el["@type"].includes("http://data.deichman.no/ontology#Work") &&
el["@id"] === workId
) {
el["@type"].push("http://data.deichman.no/ontology#WorkInFocus");
}
if (el["@type"]) {
const i = el["@type"].indexOf("http://migration.deichman.no/Work");
if (i !== -1) {
el["@type"].splice(i, 1);
}
}
return el;
});
resolve(ntdoc);
});
return el;
});
}
......@@ -373,9 +363,10 @@ function transformWork(input) {
}
}
function frameJsonLD(ntdoc) {
function frameJsonLD(data, publicationId) {
const jsonldTransformed = transformJsonLD(data, publicationId)
return new Promise((resolve, reject) => {
jsonld.frame(ntdoc, frame, (error, framedJson) => {
jsonld.frame(jsonldTransformed, frame, (error, framedJson) => {
if (error) {
reject(error);
}
......@@ -466,4 +457,4 @@ function transformPublications(publications) {
});
}
module.exports = { parseRDFtoJsonLD, frameJsonLD, addItemsToPublications };
module.exports = { frameJsonLD, addItemsToPublications };
module.exports.describeWorkViaPublication = publicationId => {
return `
PREFIX : <http://data.deichman.no/ontology#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
DESCRIBE ?publication ?work ?workContributor ?compType ?format ?subject
?genre ?instrument ?litform ?workType ?serial ?nation ?language
?pubContrib ?publicationContributor ?place ?publishedBy ?publicationPartValues
?audience ?bio ?country ?contentAdaptation ?formatAdaptation ?relatedWork ?workSeriesPart ?workSeries ?workAsSubjectAgent ?subPlace ?relworkMainEntry
?coloration ?subLanguage ?audioDescLanguage ?dubLanguage
FROM <https://katalog.deichman.no>
WHERE {
<http://data.deichman.no/publication/${publicationId}> :publicationOf ?work .
?publication :publicationOf ?work .
{
{ ?work :isRelatedTo ?related .
?related :work ?relatedWork .
OPTIONAL { ?relatedWork :contributor ?relworkContrib .
?relworkContrib a :MainEntry ; :agent ?relworkMainEntry .
}
}
UNION { ?work :contributor ?workContrib .
?workContrib a :Contribution ; :agent ?workContributor
OPTIONAL { ?workContributor :nationality ?nation } }
UNION { ?publication :hasFormatAdaptation ?formatAdaptation }
UNION { ?publication :format ?format }
UNION { ?publication :contributor ?pubContrib.
?pubContrib a :Contribution ; :agent ?publicationContributor . }
UNION { ?publication :inSerial ?serialIssue .
?serialIssue :serial ?serial . }
UNION { ?publication :hasPlaceOfPublication ?place }
UNION { ?publication :publishedBy ?publishedBy }
UNION { ?publication :language ?pubLang }
UNION { ?publication :coloration ?coloration }
UNION { ?publication :subtitlesForTheHardOfHearing ?subLanguage }
UNION { ?publication :audioDescription ?audioDescLanguage }
UNION { ?publication :dubbing ?dubLanguage }
UNION { ?publication :hasPublicationPart ?hasPublicationPart .
?hasPublicationPart a :PublicationPart;
?publicationPartProperties ?publicationPartValues . }
UNION { ?publication :publicationOf ?work . ?work :subject ?subject .
OPTIONAL { ?subject :place ?subPlace .
?subPlace a :Place
}
OPTIONAL { ?subject :contributor ?subContrib .
?subContrib a :Contribution ;
:agent ?workAsSubjectAgent } }
UNION { ?work :hasInstrumentation ?instrumentation .
?instrumentation :hasInstrument ?instrument }
UNION { ?work :genre ?genre }
UNION { ?work :literaryForm ?litform }
UNION { ?work :hasWorkType ?workType }
UNION { ?work :hasCompositionType ?compType }
UNION { ?work :biography ?bio }
UNION { ?work :language ?language }
UNION { ?work :nationality ?country }
UNION { ?work :audience ?audience}
UNION { ?work :fictionNonfiction ?fictionNonfiction }
UNION { ?work :hasContentAdaptation ?contentAdaptation }
UNION { ?work :isPartOfWorkSeries ?workSeriesPart .
?workSeriesPart a :WorkSeriesPart ;
:workSeries ?workSeries .
}
}
FILTER NOT EXISTS { ?publication :deleted ?deleted }
}
`;
};
module.exports.decorateRelation = (publicationId, relationURI) => {
return `
PREFIX : <http://data.deichman.no/ontology#>
SELECT ?mediaType ?lang ?format ?targetPub ?targetMediaType ?targetLang ?targetFormat ?targetMainTitle ?targetSubtitle ?targetPartNumber ?targetPartTitle ?workMainTitle
FROM <https://katalog.deichman.no>
WHERE {
OPTIONAL { <http://data.deichman.no/publication/${publicationId}> :hasMediaType ?mediaType }
OPTIONAL { <http://data.deichman.no/publication/${publicationId}> :language ?lang }
OPTIONAL { <http://data.deichman.no/publication/${publicationId}> :format ?format }
<${relationURI}> :mainTitle ?workMainTitle .
OPTIONAL {
?targetPub :publicationOf <${relationURI}> ; :mainTitle ?targetMainTitle .
OPTIONAL { ?targetPub :subtitle ?targetSubtitle }
OPTIONAL { ?targetPub :partNumber ?targetPartNumber }
OPTIONAL { ?targetPub :partTitle ?targetPartTitle }
OPTIONAL { ?targetPub :hasMediaType ?targetMediaType }
OPTIONAL { ?targetPub :language ?targetLang }
OPTIONAL { ?targetPub :format ?targetFormat }
FILTER NOT EXISTS {
?targetPub :deleted ?deleted .
}
}
}`;
};
module.exports.recordIdsByWorkViaPublication = publicationId => {
return `
SELECT ?recordId
FROM <https://katalog.deichman.no>
WHERE {
<http://data.deichman.no/publication/${publicationId}> <http://data.deichman.no/ontology#publicationOf> ?work .
?pub <http://data.deichman.no/ontology#publicationOf> ?work ;
<http://data.deichman.no/ontology#recordId> ?recordId .
}`;