[J-Stage.js]: Fix cases when no pdf is available (#1125)

Also fixing the extraction of the abstract and updating the structure of the translator in general. Add a new test case.
zotero · Aug 27, 2016 · 055fa204ea673c79236dc90d0860ff43afb202f5 · 055fa20
1 parent d53b3cf
commit 055fa204ea673c79236dc90d0860ff43afb202f5
Unified Split

Showing with 117 additions and 58 deletions.

+117 −58 J-Stage.js
diff --git a/J-Stage.js b/J-Stage.js
@@ -3,13 +3,13 @@
 	"label": "J-Stage",
 	"creator": "Sebastian Karcher",
 	"target": "^https?://www\\.jstage\\.jst\\.go\\.jp/",
-  
-	"minVersion": "2.1.9",
+  
+	"minVersion": "3.0",
 	"maxVersion": "",
 	"priority": 100,
 	"inRepository": true,
 	"translatorType": 4,
 	"browserSupport": "gcsibv",
-  
-	"lastUpdated": "2014-03-19 01:59:25"
+  
+	"lastUpdated": "2016-08-27 21:45:11"
 }
 /*
@@ -33,6 +33,7 @@
 	***** END LICENSE BLOCK *****
 */
+  
+
 function detectWeb(doc, url) {
 	if (url.match(/\.jst\.go\.jp\/article\//)) {
 		return "journalArticle";
@@ -41,51 +42,57 @@ function detectWeb(doc, url) {
 	}
 }
+  
+
+  
+function getSearchResults(doc, checkOnly) {
+  
+	var items = {};
+  
+	var found = false;
+  
+	var rows = ZU.xpath(doc, '//div[contains(@class, "contents_detail")]/div/a|//h3[@class="mod-item-heading"]/a');
+  
+	for (var i=0; i<rows.length; i++) {
+  
+		var href = rows[i].href;
+  
+		var title = ZU.trimInternal(rows[i].textContent);
+  
+		if (!href || !title) continue;
+  
+		if (checkOnly) return true;
+  
+		found = true;
+  
+		items[href] = title;
+  
+	}
+  
+	return found ? items : false;
+  
+}
+  
+
+  
+
 function doWeb(doc, url) {
-  
-	var articles = new Array();
 	if (detectWeb(doc, url) == "multiple") {
-  
-		var items = {};
-  
-		var titles = doc.evaluate('//div[contains(@class, "contents_detail")]/div/a|//h3[@class="mod-item-heading"]/a', doc, null, XPathResult.ANY_TYPE, null);
-  
-		var title;
-  
-		while (title = titles.iterateNext()) {
-  
-			items[title.href] = title.textContent;
-  
-		}
-  
-		Zotero.selectItems(items, function (items) {
+  
+		Zotero.selectItems(getSearchResults(doc, false), function (items) {
 			if (!items) {
 				return true;
 			}
+  
+			var articles = [];
 			for (var i in items) {
 				articles.push(i);
 			}
-  
-			Zotero.Utilities.processDocuments(articles, scrape, function () {
-  
-				Zotero.done();
-  
-			});
-  
-			Zotero.wait();
+  
+			ZU.processDocuments(articles, scrape);
 		});
 	} else {
 		scrape(doc, url);
 	}
 }
+  
+
 // help function
 function scrape(doc, url) {
 	//get abstract and tags from article plage
 	//the xpaths aren't great , but seem reliable across pages
-  
-	var pdfurl = ZU.xpath(doc, '//li[@class="icon-pdf"]/a/@href');
-  
-	pdfurl = "https://www.jstage.jst.go.jp" + pdfurl[1].textContent;
-  
-	Z.debug(pdfurl)
-  
-	var abs = ZU.xpathText(doc, '//div[@class="mod-section"]/p[@class="normal"]') //.replace(/\n/g, "")
-  
-	var tags = ZU.xpathText(doc, '//p[contains(@class, "keywords")]')
+  
+	var abs = ZU.xpathText(doc, '//div[@class="mod-section"]/*[contains(@class, "normal")]');
+  
+	var tags = ZU.xpathText(doc, '//p[contains(@class, "keywords")]');
 	if (tags){
-  
-	tags = tags.replace(/Keywords:/, "").split(/\s*,\s*/);
-  
-	for (i in tags) {
-  
-		tags[i] = ZU.trimInternal(tags[i]);
-  
-	}
+  
+		tags = tags.replace(/Keywords:/, "").split(/\s*,\s*/);
+  
+		for (var i=0; i<tags.length; i++) {
+  
+			tags[i] = ZU.trimInternal(tags[i]);
+  
+		}
 	}
+  
+
 	//get BibTex Link
 	var bibtexurl = ZU.xpathText(doc, '//li/a[contains(text(), "BibTeX")]/@href');
-  
-	Z.debug(bibtexurl)
-  
-	Zotero.Utilities.HTTP.doGet(bibtexurl, function (text) {
+  
+	ZU.doGet(bibtexurl, function (text) {
 		var bibtex = text;
 		//Zotero.debug(bibtex)
 		var translator = Zotero.loadTranslator("import");
@@ -94,29 +101,35 @@ function scrape(doc, url) {
 		translator.setHandler("itemDone", function (obj, item) {
 			if (abs) item.abstractNote = abs;
 			if (tags) item.tags = tags;
-  
-			for (i in item.creators) {
+  
+			for (var i=0; i<item.creators.length; i++) {
 				if (item.creators[i].lastName && item.creators[i].lastName == item.creators[i].lastName.toUpperCase()) {
-  
-					item.creators[i].lastName = Zotero.Utilities.capitalizeTitle(item.creators[i].lastName.toLowerCase(), true);
+  
+					item.creators[i].lastName = ZU.capitalizeTitle(item.creators[i].lastName.toLowerCase(), true);
 				}
 				if (item.creators[i].firstName && item.creators[i].firstName == item.creators[i].firstName.toUpperCase()) {
-  
-					item.creators[i].firstName = Zotero.Utilities.capitalizeTitle(item.creators[i].firstName.toLowerCase(), true);
+  
+					item.creators[i].firstName = ZU.capitalizeTitle(item.creators[i].firstName.toLowerCase(), true);
 				}
 			}
 			if (item.title == item.title.toUpperCase()) {
-  
-				item.title = Zotero.Utilities.capitalizeTitle(item.title.toLowerCase(), true);
+  
+				item.title = ZU.capitalizeTitle(item.title.toLowerCase(), true);
 			}
 			if (item.publicationTitle == item.publicationTitle.toUpperCase()) {
-  
-				item.publicationTitle = Zotero.Utilities.capitalizeTitle(item.publicationTitle.toLowerCase(), true);
+  
+				item.publicationTitle = ZU.capitalizeTitle(item.publicationTitle.toLowerCase(), true);
 			}
-  
-			item.attachments = [{
-  
-				url: item.url,
-  
-				title: "Jstage - Snapshot",
+  
+			item.attachments.push({
+  
+				url: url,
+  
+				title: "J-Stage - Snapshot",
 				mimeType: "text/html"
-  
-			}, {
-  
-				url: pdfurl,
-  
-				title: "Jstage - Full Text PDF",
-  
-				mimeType: "application/pdf"
-  
-			}];
+  
+			});
+  
+
+  
+			var pdfurl = ZU.xpath(doc, '//li[@class="icon-pdf"]/a/@href');
+  
+			if (pdfurl.length>0) {
+  
+				pdfurl = "https://www.jstage.jst.go.jp" + pdfurl[1].textContent;
+  
+				item.attachments.push({
+  
+					url: pdfurl,
+  
+					title: "J-Stage - Full Text PDF",
+  
+					mimeType: "application/pdf"
+  
+				});
+  
+			}
 			item.complete();
 		});
 		translator.translate();
@@ -129,6 +142,7 @@ var testCases = [
 		"items": [
 			{
 				"itemType": "journalArticle",
+  
+				"title": "Application of RUSLE Model on Global Soil Erosion Estimate",
 				"creators": [
 					{
 						"firstName": "Thai Nam",
@@ -156,46 +170,91 @@ var testCases = [
 						"creatorType": "author"
 					}
 				],
-  
-				"notes": [],
-  
-				"tags": [
-  
-					"soil erosion by water",
-  
-					"the RUSLE",
-  
-					"global estimation",
-  
-					"global data sets"
-  
-				],
-  
-				"seeAlso": [],
+  
+				"date": "2001",
+  
+				"DOI": "10.2208/prohe.45.811",
+  
+				"abstractNote": "Soil erosion is one of the most serious environmental problems commonly in over the world, which is caused by both natural and human factors. It is possible to investigate the global issue on soil erosion with the development of global data sets. This research estimated global soil erosion by the RUSLE model with use of a comprehensive global data set. The accuracy of the estimate mostly depends on the available information related to the study area. Present available finest data was used in this study. As the desired objective of estimating soil erosion by water at global scale, the application of RUSLE has shown its positive applicability on large-scale estimates. The study has shown a global view of water soil erosion potential with 0.5-degree grid resolution. Regional validations and examinations have been carried out by different ways. The global mean of annual soil erosion by water was estimated as 1100 ton/ km2, which agrees with several results obtained in different regions.",
+  
+				"itemID": "2001811",
+  
+				"libraryCatalog": "J-Stage",
+  
+				"pages": "811-816",
+  
+				"publicationTitle": "Proceedings of Hydraulic Engineering",
+  
+				"volume": "45",
 				"attachments": [
 					{
-  
-						"title": "Jstage - Snapshot",
+  
+						"title": "J-Stage - Snapshot",
 						"mimeType": "text/html"
 					},
 					{
-  
-						"title": "Jstage - Full Text PDF",
+  
+						"title": "J-Stage - Full Text PDF",
 						"mimeType": "application/pdf"
 					}
 				],
-  
-				"itemID": "2001811",
-  
-				"title": "Application of RUSLE Model on Global Soil Erosion Estimate",
-  
-				"publicationTitle": "Proceedings of Hydraulic Engineering",
-  
-				"volume": "45",
-  
-				"pages": "811-816",
-  
-				"date": "2001",
-  
-				"abstractNote": "Soil erosion is one of the most serious environmental problems commonly in over the world, which is caused by both natural and human factors. It is possible to investigate the global issue on soil erosion with the development of global data sets. This research estimated global soil erosion by the RUSLE model with use of a comprehensive global data set. The accuracy of the estimate mostly depends on the available information related to the study area. Present available finest data was used in this study. As the desired objective of estimating soil erosion by water at global scale, the application of RUSLE has shown its positive applicability on large-scale estimates. The study has shown a global view of water soil erosion potential with 0.5-degree grid resolution. Regional validations and examinations have been carried out by different ways. The global mean of annual soil erosion by water was estimated as 1100 ton/ km2, which agrees with several results obtained in different regions.",
-  
-				"libraryCatalog": "J-Stage"
+  
+				"tags": [
+  
+					"global data sets",
+  
+					"global estimation",
+  
+					"soil erosion by water",
+  
+					"the RUSLE"
+  
+				],
+  
+				"notes": [],
+  
+				"seeAlso": []
 			}
 		]
 	},
 	{
 		"type": "web",
-  
-		"defer": true,
 		"url": "https://www.jstage.jst.go.jp/result?item1=4&word1=organic+agriculture+erosion",
+  
+		"defer": true,
 		"items": "multiple"
 	},
 	{
 		"type": "web",
-  
-		"defer": true,
 		"url": "https://www.jstage.jst.go.jp/browse/bpb",
+  
+		"defer": true,
 		"items": "multiple"
+  
+	},
+  
+	{
+  
+		"type": "web",
+  
+		"url": "https://www.jstage.jst.go.jp/article/jfs/114/0/114_0_280/_article/-char/ja/",
+  
+		"items": [
+  
+			{
+  
+				"itemType": "journalArticle",
+  
+				"title": "フラックスタワーデータを用いた各種植生指標の季節変化の検討",
+  
+				"creators": [
+  
+					{
+  
+						"firstName": "田中",
+  
+						"lastName": "博春",
+  
+						"creatorType": "author"
+  
+					},
+  
+					{
+  
+						"firstName": "小熊",
+  
+						"lastName": "宏之",
+  
+						"creatorType": "author"
+  
+					}
+  
+				],
+  
+				"date": "2003",
+  
+				"DOI": "10.11519/jfs.114.0.280.0",
+  
+				"abstractNote": "I.  はじめに　分光日射計データから得られる各種植生指標の季節変化を、CO2吸収量ならびに葉面積指数の季節変化と比較した。データは、国立環境研究所苫小牧フラックスリサーチサイト（カラマツ人工林）のタワーデータを用いた。・各種植生指標：全天分光日射計 英弘精機MS-131WP使用。地上高40mに設置した上向き・下向きの日積算日射量より各種植生指標値を算出。波長帯は、可視(Ch3：590-695nm≒ 赤)と近赤外(Ch5：850-1200nm)の組み合わせ[図1-a]、ならびに可視(Ch2：395-590nm≒青・緑)と 近赤外(Ch4：695-850nm)の組み合わせ[図1-b]の2通りを用いた。・CO2フラックス日中積算値：クローズドパス法非分散型赤外線分析計Li-Cor LI-6262使用。地上高27m 9:00から16:30までの30分値を加算、日中の積算値とした[図1-c]。・葉面積指数(LAI)：光合成有効放射計Li-Cor LI-190SB 地上高1.5mと40mの下向き光合成有効放射量(PAR)の日積算値の比から、Lambert-Beerの式を用いPAI(Plant Area Index)を算出。落葉期の測定値を減じLAIとした [図1-d]。II. 日中CO2フラックスと植生指標GEMIの整合性[図1-c]　Ch2とCh4から求めた植生指標GEMI(Global Environmental Monitoring Index)の季節変化と、日中積算CO2フラックスの極小値を結んだ包絡線の季節変化の間によい一致がみられた[図1-c]。特にカラマツの萌芽後のGEMI値の急増時期や、展葉に伴うGEMI値の増加傾向が、CO2フラックスの変化傾向とよく一致している。ただし紅葉期は両者は一致しない。これは、光合成活動が低下した葉が落葉せずに残るためと思われる。III.  各種植生指標の季節変化 [図1-a,b]　これに対し、植生指標としてよく用いられる正規化植生指標NDVI(Normalized Vegetation Index)は、CO2フラックスの季節変化傾向と一致しなかった。NDVIは春先の融雪に伴う値のジャンプがあり、また6__から__10月の活葉期に値がだいたい一定となる。この特徴は、Ch3とCh5から求めた図1-aの4つの植生指標も同様であった。しかし、Ch2とCh4を用いた図1-bのGEMIと、近赤外と可視の差であるDVI(Difference Vegetation Index)にはこれらの特徴がみられず、CO2フラックスの季節変化傾向と同様に萌芽後に値が急増し、6月にピークを迎えた後なだらかに減少した。IV.  葉面積指数LAIと植生指標GEMIの整合性 [図1-d] 葉面積指数(LAI)が正常値を示す、積雪期以外のLAIの季節変化を、Ch2とCh4によるGEMI（≒CO2フラックスの季節変化）と比較すると、カラマツ萌芽後の展葉期にはGEMIより1__から__2週間ほど遅れてLAIの値が増加した。タワー設置のモニタリングカメラの日々の画像の変化を見ても、カラマツの葉の色の変化が先に現れ、その後に葉が茂ってゆく様子がわかる。　萌芽後、LAIは直線的に増加するが、GEMIの増加は立ち上がりは急なものの徐々に増加量が減ってくる。これは、萌芽後LAIの増加とともに葉の相互遮蔽が生じ、下層まで届く光量が減少するため、群落全体としての光合成活動が低下することが原因と思われる。　他にも、今回の測定方法ではLAIとしてカウントされていない林床植物のCO2フラックスの影響等が想定される。＜CO2フラックス・LAIデータ提供: 産業総合技術研究所 三枝 信子・王 輝民＞",
+  
+				"itemID": "田中博春2003",
+  
+				"libraryCatalog": "J-Stage",
+  
+				"pages": "280-280",
+  
+				"publicationTitle": "日本林学会大会発表データベース",
+  
+				"volume": "114",
+  
+				"attachments": [
+  
+					{
+  
+						"title": "J-Stage - Snapshot",
+  
+						"mimeType": "text/html"
+  
+					}
+  
+				],
+  
+				"tags": [
+  
+					"CO2フラックス",
+  
+					"キーワード: 分光日射計",
+  
+					"植生指標",
+  
+					"苫小牧フラックスリサーチサイト",
+  
+					"葉面積指数"
+  
+				],
+  
+				"notes": [],
+  
+				"seeAlso": []
+  
+			}
+  
+		]
 	}
 ]
 /** END TEST CASES **/