Skip to content

GitLab

Commit a9eed2fe authored by Petter Goksøyr Åsen's avatar Petter Goksøyr Åsen
Browse files

DEICH-5603 fetch work data from euler, and not directly from virtuoso

parent 099fc303
Hide whitespace changes
Inline Side-by-side
deichman.no/server/routes/resources.js
View file @ a9eed2fe
const routes = require("express").Router();
const fetch = require("isomorphic-unfetch");
const uniqBy = require("lodash/uniqBy");
const sparqlQueries = require("../utils/sparqlQueries");
const {
parseRDFtoJsonLD,
frameJsonLD,
addItemsToPublications
} = require("../utils/resourceHelpers");
......@@ -14,7 +12,6 @@ const logger = require("../../logger")(__filename);
const CALL_ID_HEADER = "Deichman-CallID";
const TIMEOUT_MS = 5000;
const kohaEndpoint = process.env.INTERNAL_URL_KOHA;
const virtuosoEndpoint = process.env.SPARQL_ENDPOINT;
const { INTERNAL_URL_EULER, INTERNAL_URL_SIBYL } = process.env;
// ----- Shared promises -----
......@@ -24,21 +21,21 @@ const publicationLinkScore = binding => {
if (
binding.mediaType &&
binding.targetMediaType &&
binding.mediaType.value === binding.targetMediaType.value
binding.mediaType === binding.targetMediaType
) {
score++;
}
if (
binding.format &&
binding.targetFormat &&
binding.format.value === binding.targetFormat.value
binding.format === binding.targetFormat
) {
score++;
}
if (
binding.lang &&
binding.targetLang &&
binding.lang.value === binding.targetLang.value
binding.lang === binding.targetLang
) {
score += 3;
}
......@@ -46,13 +43,9 @@ const publicationLinkScore = binding => {
};
async function decorateRelation(key, publicationId, relation, deichmanCallId) {
const query = sparqlQueries.decorateRelation(publicationId, relation.uri);
return fetch(virtuosoEndpoint, {
method: "POST",
body: `query=${encodeURIComponent(query)}`,
const url = `${INTERNAL_URL_EULER}/api/authorities/decorateRelation?id=${publicationId}&relation=${encodeURIComponent(relation.uri)}`;
return fetch(url, {
headers: {
Accept: "application/sparql-results+json",
"Content-Type": "application/x-www-form-urlencoded",
"Deichman-CallID": deichmanCallId
}
})
......@@ -64,26 +57,26 @@ async function decorateRelation(key, publicationId, relation, deichmanCallId) {
})
.then(json => {
let selected = false;
json.results.bindings.forEach(binding => {
(json || []).forEach(binding => {
const score = publicationLinkScore(binding);
if (!selected || score > selected.score) {
if (binding.targetPub) {
selected = {
score,
publicationId: binding.targetPub.value.replace(
publicationId: binding.targetPub.replace(
"http://data.deichman.no/publication/",
""
),
mainTitle: binding.targetMainTitle.value
mainTitle: binding.targetMainTitle
};
if (binding.targetSubtitle) {
selected.subtitle = binding.targetSubtitle.value;
selected.subtitle = binding.targetSubtitle;
}
if (binding.targetPartNumber) {
selected.partNumber = binding.targetPartNumber.value;
selected.partNumber = binding.targetPartNumber;
}
if (binding.targetPartTitle) {
selected.partTitle = binding.targetPartTitle.value;
selected.partTitle = binding.targetPartTitle;
}
}
}
......@@ -95,11 +88,11 @@ async function decorateRelation(key, publicationId, relation, deichmanCallId) {
relation.partTitle = selected.partTitle;
if (
!selected &&
json.results.bindings.length > 0 &&
json.results.bindings[0].workMainTitle
json.length > 0 &&
json[0].workMainTitle
) {
// Related work has no publications yet, so we use the title from the work.
relation.mainTitle = json.results.bindings[0].workMainTitle.value;
relation.mainTitle = json[0].workMainTitle;
}
return { key, relation };
});
......@@ -151,69 +144,36 @@ function addRecommendationsAndLikesToPublication(
}
const getWorkDataByPublication = (publicationId, deichmanCallId) => {
const query = sparqlQueries.describeWorkViaPublication(publicationId);
return fetch(virtuosoEndpoint, {
method: "POST",
body: `query=${encodeURIComponent(query)}`,
headers: {
Accept: "application/n-triples;charset=utf-8",
"Content-Type": "application/x-www-form-urlencoded",
"Deichman-CallID": deichmanCallId
}
})
.then(res => {
if (res.status === 200) {
return res.text();
}
const url = `${INTERNAL_URL_EULER}/api/authorities/expandedWorkByPublication/${publicationId}`;
return fetch(url, {
method: "GET",
headers: { "Deichman-CallID": deichmanCallId }
}).then(res => {
if (res.status !== 200) {
return Promise.reject(
new Error(`error fetching work ${publicationId}: ${res.statusText}`)
);
})
.then(ntdata => {
if (!ntdata || ntdata.length < 100) {
// virtuoso returns one line with comment: # Empty NT
}
return res.json()
.then(jsonld => {
if (jsonld.length === 0 ) { // empty json-ld
return Promise.reject(
new Error(`error fetching work ${publicationId}: 404 not found`)
new Error(`404 publication ${publicationId} not found}`)
);
}
return parseRDFtoJsonLD(ntdata, publicationId);
return frameJsonLD(jsonld, publicationId)
})
.then(ntdoc => frameJsonLD(ntdoc));
})
};
// TODO rethink this function - we allready have the recordIds in the response from getWorkDataByPublication
const getWorkItemsByPublication = (
publicationId,
categorycode,
deichmanCallId
const getWorkItemsByRecordIds = (
recordIds,
categorycode
) => {
const query = sparqlQueries.recordIdsByWorkViaPublication(publicationId);
return fetch(virtuosoEndpoint, {
method: "POST",
body: `query=${encodeURIComponent(query)}`,
headers: {
Accept: "application/sparql-results+json",
"Content-Type": "application/x-www-form-urlencoded",
"Deichman-CallID": deichmanCallId
}
})
.then(res => {
if (res.status === 200) {
return res.json();
}
logger.info(
`ERROR fetching items for work by publication ${publicationId}: ${
res.statusText
}`,
{ publication_id: publicationId, call_id: deichmanCallId }
);
return Promise.resolve({ results: { bindings: [] } }); // We don't wan't to fail if items cannot be found
})
.then(json =>
Promise.all(
json.results.bindings.map(binding =>
return Promise.all(
recordIds.map(id =>
fetch(
`${kohaEndpoint}/api/biblio/${binding.recordId.value}/expanded${
`${kohaEndpoint}/api/biblio/${id}/expanded${
categorycode ? "?categorycode=" + categorycode : ""
}`
).then(res => {
......@@ -223,81 +183,80 @@ const getWorkItemsByPublication = (
return Promise.resolve(undefined);
})
)
).then(itemResponses => {
const itemsByRecordId = {};
const holdsByRecordId = {};
itemResponses
.filter(itemResponse => itemResponse)
.forEach(itemResponse => {
holdsByRecordId[itemResponse.biblio.biblionumber] =
itemResponse.biblio;
const items = {};
itemResponse.items.forEach(item => {
if (item.status === "Utilgjengelig") {
// Theese items should not count, or be processed further,
// as they are not available to end users.
return;
}
const newItem = {
shelfmark: item.itemcallnumber,
status: item.status,
reservable: item.reservable === 1,
branchcode: item.homebranch,
barcode: item.barcode,
notforloan:
item.status === "Ikke til hjemlån" ||
item.status === "Ikke til hjemlån (utlånt)",
notforloanAndCheckedOut:
item.status === "Ikke til hjemlån (utlånt)"
};
if (item.newLocation) {
const { livesLabel = "", coLabel = "" } = item.newLocation;
newItem.locLabel = coLabel ? coLabel : livesLabel;
if (
item.newLocation.meta.poi &&
item.newLocation.meta.poi.url
) {
newItem.poi = item.newLocation.meta.poi.url;
}
}
const key = `${newItem.branchcode}_${newItem.shelfmark}_${
newItem.locLabel
}_${newItem.notforloan}`;
).then(itemResponses => {
const itemsByRecordId = {};
const holdsByRecordId = {};
itemResponses
.filter(itemResponse => itemResponse)
.forEach(itemResponse => {
  • Torstein @Torstein ·
    Contributor

    hadde det gitt mening å flytte mer av dette ut til Euler, eller er det presentasjonslogikk?

  • Petter Goksøyr Åsen @boutros ·
    Owner

    Det er jo noe vi har snakket om, men i dag så går ingenting fra koha via euler, så vi bør vel ha en grundig diskusjon og plan først. Kanskje det gir mer mening å massere ferdig i koha sine apier

Please register or sign in to reply
holdsByRecordId[itemResponse.biblio.biblionumber] =
itemResponse.biblio;
const items = {};
itemResponse.items.forEach(item => {
if (item.status === "Utilgjengelig") {
  • Torstein @Torstein ·
    Contributor

    Hvor kommer disse strengene fra? Har vi ikke en ID eller noe flagg vi kan bruke? Dette er kode som på magisk og umerkelig vis plutselig slutter å fungerer etter min erfaring.

  • Petter Goksøyr Åsen @boutros ·
    Owner

    De kommer fra koha. Denne delen av koden har ikke blitt endret i denne omgangen, kun flyttet litt rundt på.

Please register or sign in to reply
// Theese items should not count, or be processed further,
// as they are not available to end users.
return;
}
const newItem = {
shelfmark: item.itemcallnumber,
status: item.status,
reservable: item.reservable === 1,
branchcode: item.homebranch,
barcode: item.barcode,
notforloan:
item.status === "Ikke til hjemlån" ||
item.status === "Ikke til hjemlån (utlånt)",
notforloanAndCheckedOut:
item.status === "Ikke til hjemlån (utlånt)"
};
if (!items[key]) {
newItem.available = 0;
newItem.total = 0;
items[key] = newItem;
}
if (item.newLocation) {
const { livesLabel = "", coLabel = "" } = item.newLocation;
newItem.locLabel = coLabel ? coLabel : livesLabel;
if (
newItem.status === "Ledig" ||
(newItem.notforloan && !newItem.notforloanAndCheckedOut)
item.newLocation.meta.poi &&
  • Torstein @Torstein ·
    Contributor

    LItt magisk... Kan du legge til en kommentar på hva denne sjekken gjør?

  • Petter Goksøyr Åsen @boutros ·
    Owner

    Dette er gammel kode, kun whitespace endringer som ser litt rare ut her tror jeg. Har ikke endret på dette

  • Torstein @Torstein ·
    Contributor

    Vet du hva koden gjør?

  • Petter Goksøyr Åsen @boutros ·
    Owner

    Infløkte regler for hva som bestemmer om et eksemplar er tilgjengelig eller ei tror jeg? Burde vel skjedd på et lavere lag (koha). Det ser litt rarere ut i denne diffen en i filen slik den er. Det virker som jeg har endra på mye greier men det er bare indenteringen som er endra

Please register or sign in to reply
item.newLocation.meta.poi.url
) {
items[key].available++;
}
items[key].total++;
if (newItem.reservable) {
items[key].reservable = true;
newItem.poi = item.newLocation.meta.poi.url;
}
});
itemsByRecordId[itemResponse.biblio.biblionumber] = items;
}
const key = `${newItem.branchcode}_${newItem.shelfmark}_${
newItem.locLabel
}_${newItem.notforloan}`;
if (!items[key]) {
newItem.available = 0;
newItem.total = 0;
items[key] = newItem;
}
if (
newItem.status === "Ledig" ||
(newItem.notforloan && !newItem.notforloanAndCheckedOut)
) {
items[key].available++;
}
items[key].total++;
if (newItem.reservable) {
items[key].reservable = true;
}
});
const publications = {};
Object.keys(itemsByRecordId).forEach(key => {
publications[key] = {
items: Object.keys(itemsByRecordId[key]).map(
key2 => itemsByRecordId[key][key2]
)
};
});
Object.keys(holdsByRecordId).forEach(recordId => {
publications[recordId].numHolds = holdsByRecordId[recordId].numholds;
itemsByRecordId[itemResponse.biblio.biblionumber] = items;
});
return publications;
})
);
const publications = {};
Object.keys(itemsByRecordId).forEach(key => {
publications[key] = {
items: Object.keys(itemsByRecordId[key]).map(
key2 => itemsByRecordId[key][key2]
)
};
});
Object.keys(holdsByRecordId).forEach(recordId => {
publications[recordId].numHolds = holdsByRecordId[recordId].numholds;
});
return publications;
})
};
// Get objects referring to a list of publications, if they exist
......@@ -503,16 +462,11 @@ routes.get("/publication/:publicationId/detailed", async (req, res, next) => {
call_id: deichmanCallId
});
try {
const results = await Promise.all([
getWorkDataByPublication(req.params.publicationId, deichmanCallId),
getWorkItemsByPublication(
req.params.publicationId,
req.query.categorycode,
deichmanCallId
)
]);
const workWithoutLinks = await getWorkDataByPublication(req.params.publicationId, deichmanCallId);
const recordIds = workWithoutLinks.publications.map(p => p.recordId);
const items = await getWorkItemsByRecordIds(recordIds, req.query.categorycode);
const work = await addWorkLinks(
results[0],
workWithoutLinks,
req.params.publicationId,
deichmanCallId
);
......@@ -557,7 +511,7 @@ routes.get("/publication/:publicationId/detailed", async (req, res, next) => {
return true;
});
const likes = recommendationsAndLikesResults[2];
const publication = addItemsToPublications(work, results[1]);
const publication = addItemsToPublications(work, items);
const publicationWithRecommendationsAndLikes = addRecommendationsAndLikesToPublication(
publication,
filteredRecommendations,
......
deichman.no/server/utils/resourceHelpers.js
View file @ a9eed2fe
......@@ -265,54 +265,44 @@ function addItemsToPublications(work, items) {
};
}
function parseRDFtoJsonLD(ntdata, publicationId) {
return new Promise((resolve, reject) => {
jsonld.fromRDF(ntdata, { format: "application/nquads" }, (error, ntdoc) => {
if (error) {
reject(error);
}
function transformJsonLD(data, publicationId) {
// First we need to find the workId
let workId;
for (let i = 0; i < data.length; i++) {
const el = data[i];
if (
el["@id"] ===
`http://data.deichman.no/publication/${publicationId}` &&
el["http://data.deichman.no/ontology#publicationOf"] !== ""
) {
workId =
el["http://data.deichman.no/ontology#publicationOf"][0]["@id"];
break;
// TODO handle bad data we have some publications with multiple works..
}
}
// First we need to find the workId
let workId;
for (let i = 0; i < ntdoc.length; i++) {
const el = ntdoc[i];
if (
el["@id"] ===
`http://data.deichman.no/publication/${publicationId}` &&
el["http://data.deichman.no/ontology#publicationOf"] !== ""
) {
workId =
el["http://data.deichman.no/ontology#publicationOf"][0]["@id"];
break;
// TODO handle bad data we have some publications with multiple works..
}
// We need to add a class to the work 'in focus', in case there are other works
// in the graph (for example work as subject of work), and use this class
// for framing.
//
// We also delete the class migration:Work, as it trips up the framing.
return data.map(el => {
if (
el["@type"] &&
el["@type"].includes("http://data.deichman.no/ontology#Work") &&
el["@id"] === workId
) {
el["@type"].push("http://data.deichman.no/ontology#WorkInFocus");
}
if (el["@type"]) {
const i = el["@type"].indexOf("http://migration.deichman.no/Work");
if (i !== -1) {
el["@type"].splice(i, 1);
}
}
// We need to add a class to the work 'in focus', in case there are other works
// in the graph (for example work as subject of work), and use this class
// for framing.
//
// We also delete the class migration:Work, as it trips up the framing.
ntdoc = ntdoc.map(el => {
if (
el["@type"] &&
el["@type"].includes("http://data.deichman.no/ontology#Work") &&
el["@id"] === workId
) {
el["@type"].push("http://data.deichman.no/ontology#WorkInFocus");
}
if (el["@type"]) {
const i = el["@type"].indexOf("http://migration.deichman.no/Work");
if (i !== -1) {
el["@type"].splice(i, 1);
}
}
return el;
});
resolve(ntdoc);
});
return el;
});
}
......@@ -373,9 +363,10 @@ function transformWork(input) {
}
}
function frameJsonLD(ntdoc) {
function frameJsonLD(data, publicationId) {
const jsonldTransformed = transformJsonLD(data, publicationId)
return new Promise((resolve, reject) => {
jsonld.frame(ntdoc, frame, (error, framedJson) => {
jsonld.frame(jsonldTransformed, frame, (error, framedJson) => {
if (error) {
reject(error);
}
......@@ -466,4 +457,4 @@ function transformPublications(publications) {
});
}
module.exports = { parseRDFtoJsonLD, frameJsonLD, addItemsToPublications };
module.exports = { frameJsonLD, addItemsToPublications };
deichman.no/server/utils/sparqlQueries.js deleted 100644 → 0
View file @ 099fc303
module.exports.describeWorkViaPublication = publicationId => {
return `
PREFIX : <http://data.deichman.no/ontology#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
DESCRIBE ?publication ?work ?workContributor ?compType ?format ?subject
?genre ?instrument ?litform ?workType ?serial ?nation ?language
?pubContrib ?publicationContributor ?place ?publishedBy ?publicationPartValues
?audience ?bio ?country ?contentAdaptation ?formatAdaptation ?relatedWork ?workSeriesPart ?workSeries ?workAsSubjectAgent ?subPlace ?relworkMainEntry
?coloration ?subLanguage ?audioDescLanguage ?dubLanguage
FROM <https://katalog.deichman.no>
WHERE {
<http://data.deichman.no/publication/${publicationId}> :publicationOf ?work .
?publication :publicationOf ?work .
{
{ ?work :isRelatedTo ?related .
?related :work ?relatedWork .
OPTIONAL { ?relatedWork :contributor ?relworkContrib .
?relworkContrib a :MainEntry ; :agent ?relworkMainEntry .
}
}
UNION { ?work :contributor ?workContrib .
?workContrib a :Contribution ; :agent ?workContributor
OPTIONAL { ?workContributor :nationality ?nation } }
UNION { ?publication :hasFormatAdaptation ?formatAdaptation }
UNION { ?publication :format ?format }
UNION { ?publication :contributor ?pubContrib.
?pubContrib a :Contribution ; :agent ?publicationContributor . }
UNION { ?publication :inSerial ?serialIssue .
?serialIssue :serial ?serial . }
UNION { ?publication :hasPlaceOfPublication ?place }
UNION { ?publication :publishedBy ?publishedBy }
UNION { ?publication :language ?pubLang }
UNION { ?publication :coloration ?coloration }
UNION { ?publication :subtitlesForTheHardOfHearing ?subLanguage }
UNION { ?publication :audioDescription ?audioDescLanguage }
UNION { ?publication :dubbing ?dubLanguage }
UNION { ?publication :hasPublicationPart ?hasPublicationPart .
?hasPublicationPart a :PublicationPart;
?publicationPartProperties ?publicationPartValues . }
UNION { ?publication :publicationOf ?work . ?work :subject ?subject .
OPTIONAL { ?subject :place ?subPlace .
?subPlace a :Place
}
OPTIONAL { ?subject :contributor ?subContrib .
?subContrib a :Contribution ;
:agent ?workAsSubjectAgent } }
UNION { ?work :hasInstrumentation ?instrumentation .
?instrumentation :hasInstrument ?instrument }
UNION { ?work :genre ?genre }
UNION { ?work :literaryForm ?litform }
UNION { ?work :hasWorkType ?workType }
UNION { ?work :hasCompositionType ?compType }
UNION { ?work :biography ?bio }
UNION { ?work :language ?language }
UNION { ?work :nationality ?country }
UNION { ?work :audience ?audience}
UNION { ?work :fictionNonfiction ?fictionNonfiction }
UNION { ?work :hasContentAdaptation ?contentAdaptation }
UNION { ?work :isPartOfWorkSeries ?workSeriesPart .
?workSeriesPart a :WorkSeriesPart ;
:workSeries ?workSeries .
}
}
FILTER NOT EXISTS { ?publication :deleted ?deleted }
}
`;
};
module.exports.decorateRelation = (publicationId, relationURI) => {
return `
PREFIX : <http://data.deichman.no/ontology#>
SELECT ?mediaType ?lang ?format ?targetPub ?targetMediaType ?targetLang ?targetFormat ?targetMainTitle ?targetSubtitle ?targetPartNumber ?targetPartTitle ?workMainTitle
FROM <https://katalog.deichman.no>
WHERE {
OPTIONAL { <http://data.deichman.no/publication/${publicationId}> :hasMediaType ?mediaType }
OPTIONAL { <http://data.deichman.no/publication/${publicationId}> :language ?lang }
OPTIONAL { <http://data.deichman.no/publication/${publicationId}> :format ?format }
<${relationURI}> :mainTitle ?workMainTitle .
OPTIONAL {
?targetPub :publicationOf <${relationURI}> ; :mainTitle ?targetMainTitle .
OPTIONAL { ?targetPub :subtitle ?targetSubtitle }
OPTIONAL { ?targetPub :partNumber ?targetPartNumber }
OPTIONAL { ?targetPub :partTitle ?targetPartTitle }
OPTIONAL { ?targetPub :hasMediaType ?targetMediaType }
OPTIONAL { ?targetPub :language ?targetLang }
OPTIONAL { ?targetPub :format ?targetFormat }
FILTER NOT EXISTS {
?targetPub :deleted ?deleted .
}
}
}`;
};
module.exports.recordIdsByWorkViaPublication = publicationId => {
return `
SELECT ?recordId
FROM <https://katalog.deichman.no>
WHERE {
<http://data.deichman.no/publication/${publicationId}> <http://data.deichman.no/ontology#publicationOf> ?work .
?pub <http://data.deichman.no/ontology#publicationOf> ?work ;
<http://data.deichman.no/ontology#recordId> ?recordId .
}`;