Complete rewrite of FAZ.NET.js (zotero#1474)

Closes zotero#1470.
adam3smith · Nov 17, 2017 · 1a74e09c6bfbe687b9f326573745f7c89967e941 · 1a74e09
1 parent 432125c
commit 1a74e09c6bfbe687b9f326573745f7c89967e941
Unified Split

Showing with 121 additions and 78 deletions.

+121 −78 FAZ.NET.js
diff --git a/FAZ.NET.js b/FAZ.NET.js
@@ -1,54 +1,66 @@
 {
 	"translatorID": "4f0d0c90-5da0-11df-a08a-0800200c9a66",
 	"label": "FAZ.NET",
-  
-	"creator": "ibex, Sebastian Karcher",
+  
+	"creator": "Philipp Zumstein",
 	"target": "^https?://((www\\.)?faz\\.net/.)",
-  
-	"minVersion": "2.1",
+  
+	"minVersion": "3.0",
 	"maxVersion": "",
 	"priority": 100,
 	"inRepository": true,
 	"translatorType": 4,
 	"browserSupport": "gcsibv",
-  
-	"lastUpdated": "2016-09-19 21:07:47"
+  
+	"lastUpdated": "2017-11-11 11:24:04"
 }
 /*
-  
-	FAZ Translator - Parses FAZ articles and creates Zotero-based metadata.
-  
-	Copyright (C) 2010-2012 ibex and Sebastian Karcher
+  
+	***** BEGIN LICENSE BLOCK *****
-  
-	This program is free software: you can redistribute it and/or modify
-  
-	it under the terms of the GNU General Public License as published by
+  
+	Copyright © 2017 Philipp Zumstein
+  
+	
+  
+	This file is part of Zotero.
+  
+
+  
+	Zotero is free software: you can redistribute it and/or modify
+  
+	it under the terms of the GNU Affero General Public License as published by
 	the Free Software Foundation, either version 3 of the License, or
 	(at your option) any later version.
-  
-	This program is distributed in the hope that it will be useful,
+  
+	Zotero is distributed in the hope that it will be useful,
 	but WITHOUT ANY WARRANTY; without even the implied warranty of
 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-  
-	GNU General Public License for more details.
+  
+	GNU Affero General Public License for more details.
-  
-	You should have received a copy of the GNU General Public License
-  
-	along with this program. If not, see <http://www.gnu.org/licenses/>.
+  
+	You should have received a copy of the GNU Affero General Public License
+  
+	along with Zotero. If not, see <http://www.gnu.org/licenses/>.
+  
+
+  
+	***** END LICENSE BLOCK *****
 */
-  
-/* Zotero API */
+  
+// attr()/text() v2
+  
+function attr(docOrElem,selector,attr,index){var elem=index?docOrElem.querySelectorAll(selector).item(index):docOrElem.querySelector(selector);return elem?elem.getAttribute(attr):null;}function text(docOrElem,selector,index){var elem=index?docOrElem.querySelectorAll(selector).item(index):docOrElem.querySelector(selector);return elem?elem.textContent:null;}
-  
-function detectWeb(doc, url) {
-  
-	//Zotero.debug("ibex detectWeb URL= "+ url);
+  
+function detectWeb(doc, url) {
 	if (doc.title == "Suche und Suchergebnisse - FAZ" && getSearchResults(doc, true)) {
 		return "multiple";
-  
-	} else if (ZU.xpathText(doc, '//div[@class = "FAZArtikelEinleitung"]')) {
-  
-		return "newspaperArticle";
+  
+	} else if (text(doc, 'div.Artikel')) {
+  
+		if (text(doc, 'div.Artikel div.VideoBox')) {
+  
+			return "videoRecording";
+  
+		} else {
+  
+			return "newspaperArticle";
+  
+		}
 	}
 }
 function getSearchResults(doc, checkOnly) {
 	var items = {};
 	var found = false;
-  
-	//make sure we don't get media objects
-  
-	var rows = ZU.xpath(doc, '//div[not(descendant::span[@class="icon-play30"])]/a[@class="TeaserHeadLink"]');
+  
+	var rows = ZU.xpath(doc, '//div/a[@class="TeaserHeadLink"]');
 	for (var i=0; i<rows.length; i++) {
+  
+		// skip paywalled content
+  
+		if (ZU.xpathText(rows[i], './span[contains(@class, "fazplusIcon")]')) {
+  
+			continue;
+  
+		}
 		var href = rows[i].href;
 		var title = ZU.trimInternal(rows[i].textContent);
 		if (!href || !title) continue;
@@ -79,64 +91,39 @@ function doWeb(doc, url) {
 function scrape(doc, url) {
-  
-	var newArticle = new Zotero.Item('newspaperArticle');
-  
-	newArticle.url = url;
-  
-	newArticle.title = ZU.trimInternal(ZU.xpathText(doc, '//div[@class = "FAZArtikelEinleitung"]/h2').trim().replace(/\n/g,":")).replace(/^,/, "");
-  
-	var date = ZU.xpathText(doc, '(//span[@class="Datum"])[1]/@content');
-  
-	if (date) newArticle.date = ZU.trimInternal(date.replace(/T.+$/, ""));
-  
-	var teaser = ZU.xpathText(doc, '//div[@class="FAZArtikelEinleitung"]/p[@class = "Copy"]');
-  
-	if (teaser != null) {
-  
-		newArticle.abstractNote = Zotero.Utilities.trimInternal(teaser).replace(/^,\s*/, "");
-  
-	}
-  
-
-  
-	//some authors are in /a, some aren't we need to distinguish to get this right
-  
-	if (ZU.xpathText(doc, '//div[@class="FAZArtikelEinleitung"]/span[@class = "Autor"]/span[contains(@class, "caps")]/a') != null) {
-  
-		var xpath = '//div[@class="FAZArtikelEinleitung"]/span[@class = "Autor"]/span[contains(@class, "caps")]/a';
-  
-	} else {
-  
-		var xpath = '//div[@class="FAZArtikelEinleitung"]/span[@class ="Autor"]/span/span[contains(@class, "caps")]';
-  
-	};
-  
-	var authors = ZU.xpath(doc, xpath);
+  
+	var type = detectWeb(doc, url);
-  
-		for (i in authors) {
-  
-			newArticle.creators.push(Zotero.Utilities.cleanAuthor(authors[i].textContent, "author"));
+  
+	var translator = Zotero.loadTranslator('web');
+  
+	// Embedded Metadata
+  
+	translator.setTranslator('951c027d-74ac-47d4-a107-9c3069ab7b48');
+  
+	// translator.setDocument(doc);
+  
+
+  
+	translator.setHandler('itemDone', function (obj, item) {
+  
+		// fix authors
+  
+		item.creators = [];
+  
+		var authors = doc.querySelectorAll('.atc-Meta .atc-MetaAuthor');
+  
+		for (let i=0; i<authors.length; i++) {
+  
+			item.creators.push(ZU.cleanAuthor(authors[i].textContent, "author"));
 		}
+  
+
+  
+		var section = text(doc, '.gh-MainNav_SectionsLink-is-active');
+  
+		if (section) {
+  
+			item.section = Zotero.Utilities.trimInternal(section);
+  
+		}
+  
+		if (!item.language) {
+  
+			item.language = "de-DE";
+  
+		}
+  
+		item.ISSN = "0174-4909";
+  
+		item.runningTime = attr(doc, 'meta[itemprop="duration"]', 'content');
+  
+		item.complete();
+  
+	});
-  
-	newArticle.publicationTitle = "Frankfurter Allgemeine Zeitung";
-  
-
-  
-	var section = ZU.xpathText(doc, '//ul[@id="nav"]/li/span[@class = "Selected"]');
-  
-	if (section != null) {
-  
-		newArticle.section = Zotero.Utilities.trimInternal(section);
-  
-	}
-  
-
-  
-	var source = ZU.xpath(doc, '//div[@id="MainColumn"]/div[@class = "Article"]/p[@class = "ArticleSrc"]').innerHTML;
-  
-	if (source != null) {
-  
-		//	newArticle.extra = ZU.trimInternal(ZU.cleanTags(source));
-  
-	}
-  
-	//language
-  
-	var language = ZU.xpathText(doc, '//meta[@name="language"]/@content');
-  
-	if (language != null) newArticle.language = language;
-  
-	else newArticle.language = "de-DE";
-  
-
-  
-	newArticle.ISSN = "0174-4909";
-  
-	newArticle.attachments.push({
-  
-		title: "FAZ.NET Article Snapshot",
-  
-		mimeType: "text/html",
-  
-		url: doc.location.href,
-  
-		snapshot: true
+  
+	translator.getTranslatorObject(function(trans) {
+  
+		trans.itemType = type;
+  
+		trans.doWeb(doc, url);
 	});
-  
-	newArticle.complete();
 }
-  
-
-  
-/* There is no built-in function to count object properties which often are used as associative arrays.*/
-  
-
-  
-function countObjectProperties(obj) {
-  
-	var size = 0;
-  
-	for (var key in obj) {
-  
-		if (obj.hasOwnProperty(key)) size++;
-  
-	}
-  
-	return size;
-  
-};
 /** BEGIN TEST CASES **/
 var testCases = [
 	{
@@ -158,22 +145,45 @@ var testCases = [
 						"creatorType": "author"
 					}
 				],
-  
-				"date": "2011-06-13",
+  
+				"date": "2011-06-13T06:00:00+0200",
 				"ISSN": "0174-4909",
 				"abstractNote": "Wissenschaft hat eine Geschichte, wie kann sie dann aber rational sein? Im Briefwechsel zwischen Ludwik Fleck und Moritz Schlick deuteten sich bereits Antworten an.",
 				"language": "de-DE",
-  
-				"libraryCatalog": "FAZ.NET",
-  
-				"publicationTitle": "Frankfurter Allgemeine Zeitung",
+  
+				"libraryCatalog": "www.faz.net",
+  
+				"publicationTitle": "FAZ.NET",
 				"shortTitle": "Wissenschaftsphilosophie",
-  
-				"url": "http://www.faz.net/sonntagszeitung/wissenschaft/wissenschaftsphilosophie-krumme-wege-der-vernunft-1654864.html",
+  
+				"url": "http://www.faz.net/1.654864",
 				"attachments": [
 					{
-  
-						"title": "FAZ.NET Article Snapshot",
-  
-						"mimeType": "text/html",
-  
-						"snapshot": true
+  
+						"title": "Snapshot"
+  
+					}
+  
+				],
+  
+				"tags": [
+  
+					{
+  
+						"tag": "Ludwik Fleck"
+  
+					},
+  
+					{
+  
+						"tag": "Moritz"
+  
+					},
+  
+					{
+  
+						"tag": "Moritz Schlick"
+  
+					},
+  
+					{
+  
+						"tag": "Paul Feyerabend"
+  
+					},
+  
+					{
+  
+						"tag": "Schlick"
+  
+					},
+  
+					{
+  
+						"tag": "Springer-Verlag"
+  
+					},
+  
+					{
+  
+						"tag": "Thomas S. Kuhn"
+  
+					},
+  
+					{
+  
+						"tag": "Wissenschaft"
 					}
 				],
-  
-				"tags": [],
 				"notes": [],
 				"seeAlso": []
 			}
@@ -183,6 +193,39 @@ var testCases = [
 		"type": "web",
 		"url": "http://www.faz.net/suche/?query=argentinien&suchbegriffImage.x=0&suchbegriffImage.y=0&resultsPerPage=20",
 		"items": "multiple"
+  
+	},
+  
+	{
+  
+		"type": "web",
+  
+		"url": "http://www.faz.net/aktuell/sport/tango-taenzer-kaempfen-in-buenos-aires-um-den-weltmeister-titel-15155586.html",
+  
+		"items": [
+  
+			{
+  
+				"itemType": "videoRecording",
+  
+				"title": "Argentinien: Tango-Tänzer kämpfen in Buenos Aires um den Weltmeister-Titel",
+  
+				"creators": [],
+  
+				"date": "2017-08-17T11:56:48+0200",
+  
+				"abstractNote": "Mehr als 1200 Tänzer aus 48 Ländern tanzen in Argentinien um den Weltmeister-Titel. Das Finale ist am 22. August.",
+  
+				"language": "de-DE",
+  
+				"libraryCatalog": "www.faz.net",
+  
+				"runningTime": "58",
+  
+				"shortTitle": "Argentinien",
+  
+				"url": "http://www.faz.net/aktuell/sport/tango-taenzer-kaempfen-in-buenos-aires-um-den-weltmeister-titel-15155586.html",
+  
+				"attachments": [
+  
+					{
+  
+						"title": "Snapshot"
+  
+					}
+  
+				],
+  
+				"tags": [
+  
+					{
+  
+						"tag": "Argentinien"
+  
+					},
+  
+					{
+  
+						"tag": "Buenos Aires"
+  
+					}
+  
+				],
+  
+				"notes": [],
+  
+				"seeAlso": []
+  
+			}
+  
+		]
 	}
 ]
 /** END TEST CASES **/