Files
rappaurio-sae501_502/app/node_modules/cejs/application/net/Ajax.js
2023-09-25 13:27:24 +02:00

3530 lines
114 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

/**
* @name CeL function for Ajax (Asynchronous JavaScript and XML)
* @fileoverview 本檔案包含了模擬WWW客戶端發送HTTP/HTTPS請求用的 functions。
* @since 2015/1/1
*/
'use strict';
// 'use asm';
// --------------------------------------------------------------------------------------------
// 不採用 if 陳述式,可以避免 Eclipse JSDoc 與 format 多縮排一層。
typeof CeL === 'function' && CeL.run({
// module name
name : 'application.net.Ajax',
// Promise for fetch()
require : 'data.code.compatibility.'
// library_namespace.copy_properties()
+ '|data.native'
// MIME_of()
+ '|application.net.MIME.'
// for CeL.to_file_name(), CeL.URI, CeL.Search_parameters
+ '|application.net.',
// 設定不匯出的子函式。
// no_extend : '*',
// 為了方便格式化程式碼,因此將 module 函式主體另外抽出。
code : module_code
});
function module_code(library_namespace) {
function time_message(millisecond) {
if (library_namespace.age_of) {
return library_namespace.age_of(0, millisecond, {
digits : 1
});
}
return millisecond % 1000 === 0 ? millisecond / 1000 + 's'
: millisecond + 'ms';
}
// "Error: socket hang up" {code: 'ECONNRESET'}
// "Error: connect ETIMEDOUT 1.1.1.1:80"
// {errno:'ETIMEDOUT', code: 'ETIMEDOUT', address: '125.89.70.31', port:80 }
// Error: connect ECONNREFUSED 127.0.0.1:443
// "Error: read ECONNRESET"
// {errno: 'ECONNRESET', code: 'ECONNRESET', syscall: 'read'}
// Error: getaddrinfo ENOTFOUND domain
// ERROR_BAD_STSTUS
// Error: Timeout 30s
function localize_error(error) {
var message = String(error);
if (library_namespace.gettext) {
// 處理特別的錯誤訊息。
var matched = message
.match(/^(Error: (?:(?:connect|getaddrinfo) E[A-Z]+|Timeout) )(.+)$/);
if (matched) {
message = [ matched[1] + '%1', matched[2] ];
}
message = Array.isArray(message)
// gettext_config:{"id":"error-connect-etimedout-$1","mark_type":"combination_message_id"}
// gettext_config:{"id":"error-connect-econnrefused-$1","mark_type":"combination_message_id"}
// gettext_config:{"id":"error-getaddrinfo-enotfound-$1","mark_type":"combination_message_id"}
// gettext_config:{"id":"error-timeout-$1","mark_type":"combination_message_id"}
? library_namespace.gettext.apply(null, message)
// gettext_config:{"id":"error-socket-hang-up","mark_type":"combination_message_id"}
// gettext_config:{"id":"error-read-econnreset","mark_type":"combination_message_id"}
// gettext_config:{"id":"error-write-econnaborted","mark_type":"combination_message_id"}
// gettext_config:{"id":"error-unexpected-end-of-file","mark_type":"combination_message_id"}
: library_namespace.gettext(message);
}
return message;
}
var
/** {Number}未發現之index。 const: 基本上與程式碼設計合一,僅表示名義,不可更改。(=== -1) */
NOT_FOUND = ''.indexOf('_');
/**
* null module constructor
*
* @class web Ajax 的 functions
*/
var _// JSDT:_module_
= function() {
// null module constructor
};
/**
* for JSDT: 有 prototype 才會將之當作 Class
*/
_// JSDT:_module_
.prototype = {};
// ---------------------------------------------------------------------//
// XMLHttp set ajax通信処理ライブラリ ==================
/**
* <code>
to use: include in front:
way1(good: 以reg代替functionPath!):
// [function.js]_iF
// [function.js]End
way2(old):
// [function.js]getU,functionPath,'eval(getU(functionPath));'
// [function.js]End
old:
function getU(p){var o;try{o=new ActiveXObject('Microsoft.XMLHTTP');}catch(e){o=new XMLHttpRequest();}if(o)with(o){open('GET',p,false),send(null);return responseText;}}
</code>
*/
/**
* JScript or .wsh only, 能 encode.
*
* @param {String}page_url
* page url
* @param {String}[charset]
* character encoding of HTML web page. e.g., 'UTF-8', big5,
* euc-jp, ...
* @param POST_text
* POST text
*
* @returns {String}
* @see http://neural.cs.nthu.edu.tw/jang/books/asp/getWebPage.asp?title=10-1%20%E6%8A%93%E5%8F%96%E7%B6%B2%E9%A0%81%E8%B3%87%E6%96%99
*/
function get_page(page_url, charset, POST_text) {
try {
// may cause error
var X = new ActiveXObject('Microsoft.XMLHTTP'), AS;
X.open(POST_text ? 'POST' : 'GET', page_url, false);
// POST need this
X.setRequestHeader("Content-Type",
"application/x-www-form-urlencoded");
// Download the file
X.send(POST_text || null);
AS = new ActiveXObject("ADODB.Stream");
// 可同時進行讀寫
AS.Mode = 3;
// 以二進位方式操作
AS.Type = 1;
// 開啟物件
AS.Open();
// 將 binary 的資料寫入物件內 may error
AS.Write(X.responseBody);
AS.Position = 0;
// 以文字模式操作
AS.Type = 2;
// 設定編碼方式
if (charset)
AS.Charset = charset;
// 將物件內的文字讀出
X = AS.ReadText();
// Release memory. 釋放被占用的記憶體.
AS = null;
return X;
} catch (e) {
library_namespace.warn('get_page: ' + e.message);
}
}
// ---------------------------------------------------------------------//
var KEY_URL = typeof Symbol === 'function' ? Symbol('URL')
: '\0URL to fetch';
/**
*
* @param URL_to_fetch
* @param search
* @param hash
* @returns
*
* @inner
*/
function set_parameters_and_hash(URL_to_fetch, search, hash) {
// URL_to_fetch = library_namespace.URI(URL_to_fetch);
// assert: library_namespace.is_URI(URL_to_fetch)
if (hash || hash === '') {
if (Object.defineProperty[KEY_not_native] && !/^#/.test(hash))
hash = '#' + hash;
URL_to_fetch.hash = hash;
}
URL_to_fetch.search_params.set_parameters(search);
// console.trace(URL_to_fetch.toString(charset));
return URL_to_fetch;
}
function normalize_URL_to_fetch(URL_to_fetch, charset, options) {
// console.trace(URL_to_fetch);
// https://developer.mozilla.org/en-US/docs/Web/API/URL
// [ origin + pathname, search, hash ]
// hrer = [].join('')
if (Array.isArray(URL_to_fetch)) {
URL_to_fetch = set_parameters_and_hash(library_namespace
.URI(URL_to_fetch[0]), URL_to_fetch[1], URL_to_fetch[2]);
if (charset)
URL_to_fetch.charset = charset;
} else {
// 當輸入 {URL} 時node_https.request() 會將 {URL} 轉成
// {Object}options不會考慮額外選項 (headers, ...)。
// 且必須處理 charset乾脆直接將 {URL} 轉成尋常 plain object / {URI}。
// https://nodejs.org/api/http.html#http_http_request_url_options_callback
// If url is a string, it is automatically parsed with new URL(). If
// it is a URL object, it will be automatically converted to an
// ordinary options object.
// console.log(URL_to_fetch);
URL_to_fetch = library_namespace.URI(URL_to_fetch, null, {
charset : charset
});
// console.trace(URL_to_fetch);
}
// assert: library_namespace.is_URI(URL_to_fetch)
// console.trace(URL_to_fetch);
if (options.search || options.hash) {
URL_to_fetch = set_parameters_and_hash(URL_to_fetch,
options.search, options.hash);
}
library_namespace.debug({
T : [ 'Fetching URL: %1', '{' + (typeof URL_to_fetch) + '} ['
//
+ (typeof URL_to_fetch === 'string' ? URL_to_fetch
//
: URL_to_fetch && URL_to_fetch[KEY_URL]
//
|| URL_to_fetch.toString(charset)) + ']' ]
}, 1, 'normalize_URL_to_fetch');
return URL_to_fetch;
}
if (false)
// default arguments
var get_URL_arguments = {
URL : '',
charset : '',
// HTTP方法如"GET", "POST", HEAD, "PUT", "DELETE"等。
method : 'GET',
post_data : {},
async : true,
// user name. 驗證用使用者名稱。
user : '',
// 驗證用密碼。
password : '',
// header
headers : {
contentType : 'text/xml'
},
// location.search
search : {
contentType : 'text/xml'
},
// location.hash
hash : '',
mime : 'text/xml',
// onreadystatechange
onchange : function() {
},
timeout : 0,
onfail : function(error) {
this.status;
},
onload : function() {
}
};
// XMLHttp.readyState 所有可能的值如下:
// 0 還沒開始
// 1 讀取中 Sending Data
// 2 已讀取 Data Sent
// 3 資訊交換中 interactive: getting data
// 4 一切完成 Completed
var readyState_done = 4,
//
document_head = library_namespace.is_WWW(true)
&& (document.head || document.getElementsByTagName('head')[0]);
/**
* 讀取 URL via XMLHttpRequest。
*
* @param {String|Object}URL_to_fetch
* 欲請求之目的 URL or options
* @param {Function}[onload]
* callback when successful loaded
* @param {String}[charset]
* character encoding of HTML web page. e.g., 'UTF-8', big5,
* euc-jp, ...
* @param {String|Object}[post_data]
* text data to send when method is POST
* @param {Object}[options]
* 附加參數/設定選擇性/特殊功能與選項
*
* TODO: 代理伺服器 using proxy server
*
* @see https://developer.mozilla.org/zh-TW/docs/DOM/XMLHttpRequest
* http://msdn.microsoft.com/en-us/library/ie/ms535874.aspx
*/
function get_URL(URL_to_fetch, onload, charset, post_data, options) {
// 前導作業。
if (library_namespace.is_Object(charset)) {
post_data = charset;
charset = null;
}
// 正規化並提供可隨意改變的同內容參數,以避免修改或覆蓋附加參數。
options = library_namespace.new_options(options);
// ------------------------------------------------------
if (library_namespace.is_Object(URL_to_fetch) && URL_to_fetch[KEY_URL]) {
Object.assign(options, URL_to_fetch);
// 注意: options.onload 另有用途!
// https://xhr.spec.whatwg.org/#handler-xhr-onloadstart
// onload = options.onload || onload;
post_data = options.post || post_data;
charset = options.charset || charset;
URL_to_fetch = options[KEY_URL];
}
URL_to_fetch = normalize_URL_to_fetch(URL_to_fetch, charset, options);
// assert: library_namespace.is_URI(URL_to_fetch)
if (typeof onload === 'object') {
library_namespace.debug(
'Trying to JSONP, insert page, need callback.', 3,
'get_URL');
// library_namespace.run(URL_to_fetch);
for ( var callback_param in onload) {
library_namespace.debug('Trying ('
+ (typeof onload[callback_param]) + ') ['
+ callback_param + '] = [' + onload[callback_param]
+ ']', 3, 'get_URL');
if (callback_param
&& typeof onload[callback_param] === 'function') {
var callback_name, node = document.createElement('script');
for (charset = 0; (callback_name = 'cb' + charset) in library_namespace;)
charset++;
library_namespace[callback_name] = function(data) {
library_namespace.debug('[' + URL_to_fetch
+ ']: callback 完自動移除 .js。', 2, 'get_URL');
document_head.removeChild(node);
// Release memory. 釋放被占用的記憶體.
node = null;
delete library_namespace[callback_name];
onload[callback_param](data);
};
// callback_param: callback parameter
URL_to_fetch.search_params[callback_param] = library_namespace.Class
+ '.' + callback_name;
node.src = URL_to_fetch.toString();
library_namespace.debug('Use script node: [' + node.src
+ ']', 3, 'get_URL');
document_head.appendChild(node);
return;
}
}
library_namespace.debug('Skip JSONP. No callback specified.', 3,
'get_URL');
}
if (post_data && !options.form_data) {
post_data = library_namespace.Search_parameters(post_data)
.toString(charset);
}
if (!onload && typeof options.onchange === 'function') {
onload = function() {
options.onchange(readyState_done, XMLHttp);
};
}
if (options.async === false && onload || typeof onload !== 'function') {
onload = false;
}
/**
* The XMLHttpRequest object can't be cached.
*/
var XMLHttp = library_namespace.new_XMLHttp();
try {
// IE:404 會 throw error, timeout 除了 throw error,
// 還會 readystatechange;
// Gecko 亦會 throw error
// IE 10 中local file 光 .open() 就 throw 了。
XMLHttp.open(options.method || (post_data ? 'POST' : 'GET'),
URL_to_fetch.toString(), !!onload, options.user || '',
options.password || '');
// https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/response
// XMLHttp.responseType = 'blob';
if (options.timeout > 0 && !onload) {
// https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout
XMLHttp.timeout = options.timeout;
if (typeof options.onfail === 'function')
XMLHttp.ontimeout = function(e) {
options.onfail.call(XMLHttp, e || 'Timeout');
};
}
// TODO: 處理有 onload 下之 timeout 逾時ms數
// Ajax 程式應該考慮到 server 沒有回應時之處置
if (library_namespace.is_Object(options.headers)
// https://developer.mozilla.org/zh-TW/docs/Web/API/XMLHttpRequest/setRequestHeader
// `!!XMLHttp.setRequestHeader` will throw @ HTA (HTML Application)
&& ('setRequestHeader' in XMLHttp)) {
Object.keys(options.headers).forEach(function(key) {
XMLHttp.setRequestHeader(key, options.headers[key]);
});
}
if (options.mime) {
// ignore charset!
charset = options.mime;
} else if (charset) {
// old: 'text/xml;charset=' + charset
// 但這樣會被當作 XML 解析,產生語法錯誤。
// TODO: try:
// 'text/'+(/\.x(ht)?ml$/i.test(URL_to_fetch)?'xml':'plain')+';charset='
// + charset;
charset = 'application/json;charset=' + charset;
}
// 有些版本的 Mozilla 瀏覽器在伺服器送回的資料未含 XML mime-type
// 檔頭header時會出錯。為了避免這個問題可以用下列方法覆寫伺服器傳回的檔頭以免傳回的不是 text/xml。
// http://squio.nl/blog/2006/06/27/xmlhttprequest-and-character-encoding/
// http://www.w3.org/TR/XMLHttpRequest/ search encoding
if (charset && XMLHttp.overrideMimeType)
XMLHttp.overrideMimeType(charset);
if (onload) {
XMLHttp.onreadystatechange = function() {
if (XMLHttp.readyState === readyState_done)
return onload(XMLHttp);
if (0 < XMLHttp.readyState
&& XMLHttp.readyState < readyState_done) {
if (typeof options.onchange === 'function')
options.onchange(XMLHttp.readyState, XMLHttp);
} else if (typeof options.onfail === 'function') {
options.onfail(XMLHttp);
}
};
}
// 若檔案不存在,會 throw。
XMLHttp.send(post_data || null);
if (!onload) {
// XMLHttp.response blob
// XMLHttp.responseText 會把傳回值當字串用
// XMLHttp.responseXML 會把傳回值視為 XMLDocument 物件,而後可用 JavaScript
// DOM 相關函式處理
// IE only(?):
// XMLHttp.responseBody 以unsigned array格式表示binary data
// try{responseBody=(new
// VBArray(XMLHttp.responseBody)).toArray();}catch(e){}
// http://aspdotnet.cnblogs.com/archive/2005/11/30/287481.html
// XMLHttp.responseStream return AdoStream
return XMLHttp.responseText;
}
} catch (e) {
library_namespace.error(e);
if (typeof options.onfail === 'function') {
options.onfail(XMLHttp, e);
} else if (onload) {
onload(undefined, e);
}
}
}
_.get_URL = get_URL;
// TODO: 處理 multiple requests
function get_URLs() {
}
// ----------------------------------------------------
var is_nodejs = library_namespace.platform.nodejs;
/**
* <code>
// "file": keyword for "Content-Disposition: file;"
{type:'jpg',image:{file:'fn1.jpg'}}
// will fetch url first.
{type:'jpg',image:{url:'http://host/'}}
{type:'jpg',image:{file:'fn1.jpg',type:'image/jpeg'}}
// Array: use "Content-Type: multipart/mixed;"
{type:'jpg',images:[{file:'fn1.jpg'},{file:'fn2.jpg'}]}
{type:'jpg',images:[{file:'fn1.jpg'},{file:'fn2.jpg'}],docs:[{file:'fn1.txt'},{file:'fn2.txt'}]}
{type:'jpg',images:[{file:'fn1.jpg',type:'image/jpeg'},{file:'fn1.txt',type:'text/plain'}]}
</code>
*/
// should be CRLF
// @see https://tools.ietf.org/html/rfc7578#section-4.1
var form_data_new_line = '\r\n';
function form_data_to_Array(is_slice) {
if (this.generated) {
return this.generated;
}
var boundary = '--' + this.boundary + form_data_new_line,
// generated raw post data
generated = this.generated = [ boundary ], content_length = boundary.length;
boundary = form_data_new_line + boundary;
this.forEach(function(chunk, index) {
if (Array.isArray(chunk)) {
// chunk = chunk.to_Array(true);
if (!chunk.content_length) {
console.log(chunk);
throw new Error(
// gettext_config:{"id":"the-chunk-do-not-has-regular-.content_length"}
'The chunk do not has regular .content_length!');
}
content_length += chunk.content_length;
} else {
// chunk: {String} or {Buffer}
content_length += chunk.length;
}
generated.push(chunk);
if (index < this.length - 1) {
generated.push(boundary);
content_length += boundary.length;
}
}, this);
if (!(content_length > 0)) {
console.log(this);
// gettext_config:{"id":"illegal-chunk.content_length"}
throw new Error('Illegal chunk.content_length!');
}
boundary = form_data_new_line + '--' + this.boundary;
if (!is_slice) {
boundary += '--';
}
generated.push(boundary);
content_length += boundary.length;
generated.content_length = content_length;
// console.log(generated);
return generated;
}
// 選出 data.generated 不包含之 string
function give_boundary(data_Array) {
function not_includes_in(item) {
// console.trace([ typeof item, item ]);
return Array.isArray(item) ? item.every(not_includes_in)
// item: Should be {String} or {Buffer}
: !item.includes(boundary);
}
var boundary, retry_count = 0;
while (retry_count++ < 8) {
boundary = (Number.MAX_SAFE_INTEGER * Math.random())
.toString(10 + 26);
// console.log('test boundary: [' + boundary + ']');
for (var i = 1; i < boundary.length / 2 | 0; i++) {
var slice = boundary.slice(0, i);
if (boundary.lastIndexOf(slice) > 0) {
boundary = null;
break;
}
}
// assert: boundary 不自包含,例如 'aa'自包含'a''asas'自包含'as'
if (boundary) {
if (not_includes_in(data_Array)) {
data_Array.boundary = boundary;
return boundary;
}
}
}
throw new Error('give_boundary: '
// gettext_config:{"id":"retry-too-many-times"}
+ 'Retry too many times!');
}
var to_form_data_generated = {
form_data_generated : true
};
// https://github.com/form-data/form-data/blob/master/lib/form_data.js
// https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.2
// https://tools.ietf.org/html/rfc7578
// https://tools.ietf.org/html/rfc2046#section-5.1
// The "multipart" boundary delimiters and header fields are always
// represented as 7bit US-ASCII
// https://tools.ietf.org/html/rfc2049#appendix-A
// http://stackoverflow.com/questions/4238809/example-of-multipart-form-data
function to_form_data(parameters, callback, options) {
function get_file_object(value, callback, key, slice) {
var is_url, MIME_type;
if (typeof value === 'string') {
is_url = value.includes('://');
} else
// else: assert: library_namespace.is_Object(value)
if (is_url = value.url) {
value = is_url;
// is_url = true;
} else {
// .type: MIME type
MIME_type = value.type;
// value: file_path
value = value.file;
}
function push_and_callback(MIME_type, content) {
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
var headers = 'Content-Disposition: '
+ (slice ? 'file' : 'form-data; name="' + key + '"')
+ '; filename="' + encodeURIComponent(value) + '"'
+ "; filename*=UTF-8''" + encodeURIComponent(value)
+ form_data_new_line;
if (MIME_type) {
headers += 'Content-Type: ' + MIME_type
+ form_data_new_line;
}
if (is_nodejs && Buffer.isBuffer(content)) {
headers += 'Content-Transfer-Encoding: binary'
+ form_data_new_line;
}
headers += form_data_new_line;
var chunk = [ headers, content ];
// 手動設定 Content-Length。
chunk.content_length = headers.length + content.length;
// TODO: use stream
(slice || root_data).push(chunk);
callback();
}
if (!is_url) {
var content;
// read file contents
if (is_nodejs) {
try {
// get {Buffer}
content = node_fs.readFileSync(value);
} catch (e) {
// When we cannot read, it will throw now.
// e.g., Error: ENOENT: no such file or directory, open
// 'value'
}
} else {
// node.js 之下此方法不能處理 binary data。
content = library_namespace
.get_file(value/* , 'binary' */);
}
if (!content) {
library_namespace.error([ 'to_form_data: ', {
// gettext_config:{"id":"failed-to-get-file-$1"}
T : [ 'Failed to get file: [%1]', value ]
} ]);
// Skip this one.
callback();
return;
}
if (options && options.file_post_processor) {
options.file_post_processor(value, content);
}
// value: file path → file name
value = value.match(/[^\\\/]*$/)[0];
if (!MIME_type) {
MIME_type = library_namespace.MIME_of(value);
}
push_and_callback(MIME_type, content);
return;
}
library_namespace.debug({
// gettext_config:{"id":"fetching-url-$1"}
T : [ '自網路取得 URL%1', value ]
}, 1, 'to_form_data');
_.get_URL(value, function(XMLHttp, error) {
if (options && options.url_post_processor) {
options.url_post_processor(value, XMLHttp, error);
}
if (error) {
library_namespace.error([ 'to_form_data: ', {
// gettext_config:{"id":"got-error-when-retrieving-$1-$2"}
T : [ 'Got error when retrieving [%1]: %2',
//
value, localize_error(error) ]
} ]);
// Skip this one.
callback();
return;
}
// value: url → file name
value = value.replace(/[?#].*$/, '')
.match(/([^\\\/]*)[\\\/]?$/)[1];
// console.log('-'.repeat(79));
// console.log(value);
library_namespace.debug({
// gettext_config:{"id":"got-url-from-the-network-$1-$2-bytes"}
T : [ '自網路取得 URL%1%2{{PLURAL:%2|位元組}}。', value,
XMLHttp.buffer.length ]
}, 1, 'to_form_data');
push_and_callback(XMLHttp.type, XMLHttp.buffer);
}, 'buffer');
}
parameters = library_namespace.Search_parameters(parameters);
var root_data = [], keys = Object.keys(parameters), index = 0;
root_data.to_Array = form_data_to_Array;
// console.log('-'.repeat(79));
// console.log(keys);
// 因為在遇到fetch url時需要等待因此採用async。
function process_next() {
if (false) {
console.log('-'.repeat(60));
console.log('process_next: ' + index + '/' + keys.length);
console.log(root_data);
}
if (index === keys.length) {
// 決定 boundary
give_boundary(root_data);
// WARNING: 先結束作業: 生成 .to_Array()
// 才能得到 root_data.to_Array().content_length。
root_data.to_Array();
if (false) {
console.log('-'.repeat(79));
console.log(root_data);
console.log('-'.repeat(79));
console.log(root_data.to_Array().content_length);
console.log(root_data.to_Array().join(''));
throw 5;
}
callback(root_data);
return;
}
var key = keys[index++], value = parameters[key];
// console.log(key + ': ' + JSON.stringify(value));
if (Array.isArray(value)) {
// assert: is files/urls
var slice = [], item_index = 0,
//
next_item = function() {
if (item_index === value.length) {
give_boundary(slice);
var headers = 'Content-Disposition: form-data; name="'
+ key + '"' + form_data_new_line
+ 'Content-Type: multipart/mixed; boundary='
+ slice.boundary + form_data_new_line
+ form_data_new_line;
slice = form_data_to_Array.call(slice, true);
slice.unshift(headers);
slice.content_length += headers.length;
root_data.push(slice);
process_next();
} else {
get_file_object(value[item_index++], next_item,/* key */
undefined, slice);
}
};
next_item();
return;
}
if (library_namespace.is_Object(value)) {
// assert: is file/url
get_file_object(value, process_next, key);
return;
}
// 預防有 null, undefined, {Number}, true 之類。
if (!value || typeof value === 'number'
|| typeof value === 'boolean') {
value = String(value);
}
var headers;
if (is_nodejs && typeof value === 'string') {
value = Buffer.from(value, 'utf8');
headers = 'Content-Type: text/plain; charset=UTF-8'
+ form_data_new_line;
}
// 非檔案,屬於普通的表單資料。
if (!key) {
throw new Error('No key for value: ' + value);
}
if (!value && value !== '') {
// e.g., token=undefined
throw new TypeError('Invalid type of ' + key + ': '
+ typeof value);
}
// @see function push_and_callback(MIME_type, content)
headers = 'Content-Disposition: form-data; name="' + key + '"'
+ form_data_new_line + (headers || '') + form_data_new_line;
var chunk = [ headers, value ];
// 手動設定 Content-Length。
chunk.content_length = headers.length + value.length;
root_data.push(chunk);
process_next();
}
process_next();
return root_data;
}
_.to_form_data = to_form_data;
// ---------------------------------------------------------------------//
/**
* <code>
讀取URL by XMLHttpRequest
http://jck11.pixnet.net/blog/post/11630232
* 若有多行程或為各URL設定個別XMLHttp之必要請在一開始便設定deprecated_get_URL.multi_request並且別再更改。
** 在此情況下單一URL仍只能有單一個request!
** 設定 handle_function 須注意程式在等待回應時若無執行其他程式碼將自動中止!
可設定:
while(deprecated_get_URL.doing)WScript.Sleep(1); //||timeout
arguments f:{
URL:'', // The same origin policy prevents document or script loaded from one origin, from getting or setting properties from a of a document from a different origin.(http://www.mozilla.org/projects/security/components/jssec.html#sameorigin)
enc:'UTF-8', // charset: big5, euc-jp,..
fn:(handle_function), // onLoad:function(){},
method:'GET', // POST,..
sendDoc:'text send in POST,..'
async:ture/false, // true if want to asynchronous(非同期), false if synchronous(同期的,會直到readyState==4才return) http://jpspan.sourceforge.net/wiki/doku.php?id=javascript:xmlhttprequest:behaviour
user:'userName',
passwd:'****', // password
//TODO:
parameters:'~=~&~=~', // {a:1,b:2}
header:{contentType:'text/xml'},
contentType:'text/xml',
run:true/false, // do eval
update:DOMDocument, // use onLoad/onFailed to 加工 return text. onFailed(){throw;} will abort change.
interval:\d,
decay:\d, // wait decay*interval when no change
maxInterval::\d,
//insertion:top/bottom,..
onFailed:function(error){this.status;}, // onFailed.apply(XMLHttp,[XMLHttp.status])
onStateChange:function(){},
}
handle_function:
自行處理 typeof handle_function=='function':
function handle_function(error){..}
代為處理 handle_function=[d_func,0: responseText,1: responseXML]:
responseXML: http://msdn2.microsoft.com/en-us/library/ms757878.aspx
function d_func(content,headers[,XMLHttp,URL]){
if(headers){
// content,headers各為XMLHttp.responseText內容及XMLHttp.getAllResponseHeaders()其他皆可由XMLHttp取得。
}else{
// content為error
}
}
e.g., the simplest: [function(c,h){h&&alert(c);}]
)
</code>
*/
// (URL,fn) or flag URL, handle_function handle result,
// method,sendDoc,asyncFlag,userName,password
function deprecated_get_URL(f) {
var _f = arguments.callee;
if (typeof _f.XMLHttp === 'object') {
// try{_f.XMLHttp.abort();}catch(e){}
// 此時可能衝突或lose?!
_f.XMLHttp = null;
}
// 處理 arguments
if (!library_namespace.is_Object(f))
a = arguments, f = {
URL : f,
fn : a[1],
method : a[2],
sendDoc : a[3]
};
if (f.post)
f.method = 'POST', f.sendDoc = f.post;
if (!f[KEY_URL]
|| !(_f.XMLHttp = library_namespace.new_XMLHttp(f.enc,
!/\.x(?:ht)?ml$/i.test(f[KEY_URL]))))
// throw
return;
// try{_f.XMLHttp.overrideMimeType('text/xml');}catch(e){}
if (typeof f.async !== 'boolean')
// 設定f.async
f.async = f.fn ? true : false;
else if (!f.async)
f.fn = null;
else if (!f.fn)
if (typeof _f.HandleStateChange !== 'function'
|| typeof _f.HandleContent !== 'function')
// 沒有能處理的function
// throw
return;
else
// =null;
f.fn = _f.HandleContent;
if (/* typeof _f.multi_request!=='undefined'&& */_f.multi_request) {
if (!_f.q)
// queue
_f.i = {}, _f.q = [];
// ** 沒有考慮到 POST 時 URL 相同的情況!
_f.i[f[KEY_URL]] = _f.q.length;
_f.q.push({
uri : f[KEY_URL],
XMLHttp : _f.XMLHttp,
func : f.fn,
start : _f.startTime = new Date
});
} else if (_f.q && typeof _f.clean === 'function')
_f.clean();
// for Gecko Error: uncaught exception: Permission denied to call method
// XMLHttpRequest.open
if (f[KEY_URL].indexOf('://') !== NOT_FOUND
&& typeof netscape === 'object')
if (_f.asked > 2) {
_f.clean(f[KEY_URL]);
return;
} else
try {
if (typeof _f.asked === 'undefined') {
_f.asked = 0;
alert('我們需要一點權限來使用 XMLHttpRequest.open。\n* 請勾選記住這項設定的方格。');
}
netscape.security.PrivilegeManager
// UniversalBrowserAccess
.enablePrivilege('UniversalXPConnect');
} catch (e) {
_f.asked++;
_f.clean(f[KEY_URL]);
return;
}
// if(isNaN(_f.timeout))_f.timeout=300000;//5*60*1000;
try {
// IE:404會throw error, timeout除了throw error, 還會readystatechange;
// Gecko亦會throw error
try {
_f.XMLHttp.setRequestHeader("Accept-Encoding",
"gzip, deflate, br");
} catch (e) {
}
// Set header so the called script knows that it's an XMLHttpRequest
if (false)
_f.XMLHttp.setRequestHeader("X-Requested-With",
"XMLHttpRequest");
// Set the If-Modified-Since header, if ifModified mode.
if (false)
_f.XMLHttp.setRequestHeader("If-Modified-Since",
"Thu, 01 Jan 1970 00:00:00 GMT");
if (f.method === 'POST'
// &&_f.XMLHttp.setRequestHeader
) {
// use .getAttribute('method') to get 長度不一定如此
if (false)
_f.XMLHttp.setRequestHeader("Content-Length",
f.sendDoc.length);
// 有些CGI會用Content-Type測試是XMLHttp或是regular form
// It may be necessary to specify
// "application/x-www-form-urlencoded" or "multipart/form-data"
// for posted XML data to be interpreted on the server.
_f.XMLHttp.setRequestHeader('Content-Type', Array.isArray(f.fn)
&& f.fn[1] ? 'text/xml'
// application/x-www-form-urlencoded; charset=utf-8
: 'application/x-www-form-urlencoded');
}
_f.XMLHttp.abort();
_f.XMLHttp.open(f.method || 'GET', f[KEY_URL], f.async, f.user
|| null, f.passwd || null);
// alert((f.method||'GET')+','+f[KEY_URL]+','+f.async);
/**
* @see http://www.javaworld.com.tw/jute/post/view?bid=49&id=170177&sty=3&age=0&tpg=1&ppg=1
* 根據 W3C的 XMLHttpRequest 規格書上說,①在呼叫 open
* 時如果readyState是4(Loaded) ②呼叫abort之後
* ③發生其他錯誤如網路問題無窮迴圈等等則會重設所有的值。使用全域的情況就只有第一次可以執行因為之後的readyState是4所以onreadystatechange
* 放在open之前會被清空因此onreadystatechange 必須放在open之後就可以避免這個問題。
*
* 每使用一次XMLHttpRequest不管成功或失敗都要重設onreadystatechange一次。onreadystatechange
* 的初始值是 null
*
* @see http://www.xulplanet.com/references/objref/XMLHttpRequest.html
* After the initial response, all event listeners will be
* cleared. Call open() before setting new event listeners.
*/
if (f.async) {
_f.doing = (_f.doing || 0) + 1;
_f.XMLHttp.onreadystatechange = typeof f.fn === 'function'
//
? f.fn : function(e) {
_f.HandleStateChange(e, f[KEY_URL], f.fn);
}
// ||null
;
// 應加 clearTimeout( )
setTimeout('try{deprecated_get_URL.'
//
+ (_f.multi_request ? 'q[' + _f.i[f[KEY_URL]] + ']'
: 'XMLHttp')
+ '.onreadystatechange();}catch(e){}',
// 5*60*1000;
_f.timeout || 3e5);
}
_f.XMLHttp.send(f.sendDoc || null);
if (!f.fn) {
/**
* 非async(異步的)能在此就得到 response。Safari and Konqueror cannot
* understand the encoding of text files!
*
* @see http://www.kawa.net/works/js/jkl/parsexml.html
*/
// responseXML: responseXML.loadXML(text)
return _f.XMLHttp.responseText;
}
} catch (e) {
if (typeof f.fn === 'function')
f.fn(e);
else if (typeof window === 'object')
window.status = e.message;
return e;
}
}
deprecated_get_URL.timeoutCode = -7732147;
/**
* agent handle function
*
* e: object Error, handle_function: function(return text, headers,
* XMLHttpRequest object, URL) | [ function, (default|NULL:responseText,
* others:responseXML) ]
*/
deprecated_get_URL.HandleStateChange = function(e, URL, handle_function) {
var _t = 0, isOKc, m = deprecated_get_URL.multi_request, _oXMLH;
if (m)
m = deprecated_get_URL.q[isNaN(URL) ? deprecated_get_URL.i[URL]
: URL], _oXMLH = m.XMLHttp, handle_function = m.func,
URL = m.uri;
else
_oXMLH = deprecated_get_URL.XMLHttp;
if (Array.isArray(handle_function))
_t = handle_function[1], handle_function = handle_function[0];
if (!handle_function || typeof handle_function !== 'function') {
deprecated_get_URL.doing--;
deprecated_get_URL.clean(URL);
return;
}
// http://big5.chinaz.com:88/book.chinaz.com/others/web/web/xml/index1/21.htm
if (!e)
if (typeof _oXMLH === 'object' && _oXMLH) {
if (_oXMLH.parseError
&& _oXMLH/* .responseXML */.parseError.errorCode !== 0)
e = _oXMLH.parseError, e = new Error(e.errorCode, e.reason);
else if (_oXMLH.readyState === 4) {
// only if XMLHttp shows "loaded"
// condition is OK?
isOKc = _oXMLH.status;
isOKc = isOKc >= 200
&& isOKc < 300
|| isOKc === 304
|| !isOKc
&& (location.protocol === "file:" || location.protocol === "chrome:");
if (handle_function === deprecated_get_URL.HandleContent)
// handle_function.apply()
handle_function(0, isOKc, _oXMLH, URL);
else {
// handle_function.apply()
handle_function(isOKc ? _t ? _oXMLH.responseXML
// JKL.ParseXML: Safari and Konqueror cannot
// understand the encoding of text files.
: typeof window === 'object'
&& window.navigator.appVersion
.indexOf("KHTML") !== NOT_FOUND
&& !(e = escape(_oXMLH.responseText))
.indexOf("%u") !== NOT_FOUND
//
? e : _oXMLH.responseText : 0,
//
isOKc ? _oXMLH.getAllResponseHeaders() : 0, _oXMLH, URL);
}
// URL之protocol==file:
// 可能需要重新.loadXML((.responseText+'').replace(/<\?xml[^?]*\?>/,""))
// 用 .responseXML.documentElement 可調用
deprecated_get_URL.doing--;
deprecated_get_URL.clean(URL);
return;
}
} else if (new Date - (m ? m.start : deprecated_get_URL.startTime) > deprecated_get_URL.timeout)
// timeout & timeout function
// http://www.stylusstudio.com/xmldev/199912/post40380.html
// _oXMLH.abort();
e = new Error(deprecated_get_URL.timeoutCode, 'Timeout');
// alert(URL+'\n'+_t+'\n'+e+'\n'+_oXMLH.readyState+'\n'+handle_function);
if (e) {
handle_function(e, 0, _oXMLH, URL);
deprecated_get_URL.doing--;
deprecated_get_URL.clean(URL);
}// handle_function.apply(e,URL);
};
/**
* agent content handle function<br />
* 有headers時content包含回應否則content表error
*/
deprecated_get_URL.HandleContent = function(content, headers, _oXMLHttp,
URL) {
if (headers) {
// _oXMLHttp.getResponseHeader("Content-Length")
alert("URL: " + URL + "\nHeaders:\n"
+ _oXMLHttp.getAllResponseHeaders()
+ "\n------------------------\nLastModified: "
+ _oXMLHttp.getResponseHeader("Last-Modified")
+ "\nResult:\n" + _oXMLHttp.responseText.slice(0, 200));// _oXMLHttp.responseXML.xml
} else {
// error
// test時可用deprecated_get_URL.XMLHttp.open("HEAD","_URL_",true);deprecated_get_URL(url,handle_function,'HEAD',true)。
if (content instanceof Error)
alert('Error occured!\n'
+ (typeof e === 'object' && e.number ? e.number + ':'
+ e.message : e || ''));
else if (typeof _oXMLHttp === 'object' && _oXMLHttp)
alert((_oXMLHttp.status === 404 ? "URL doesn't exist!"
: 'Error occured!')
+ '\n\nStatus: '
+ _oXMLHttp.status
+ '\n'
+ _oXMLHttp.statusText);
}
};
// 在MP模式下清乾淨queue
deprecated_get_URL.clean = function(i, force) {
// multiple requests
if (force || deprecated_get_URL.multi_request)
if (!i && isNaN(i)) {
if (deprecated_get_URL.q)
for (i in deprecated_get_URL.i)
try {
deprecated_get_URL.q[deprecated_get_URL.i[i]].XMLHttp
.abort();
// deprecated_get_URL.q[deprecated_get_URL.i[i]].XMLHttp=null;
} catch (e) {
}
deprecated_get_URL.q = deprecated_get_URL.i
// =null
= 0;
} else if (!isNaN(i)
|| !isNaN(i = deprecated_get_URL.i[typeof i === 'object' ? i.uri
: i])) {
try {
deprecated_get_URL.q[i].XMLHttp.abort();
} catch (e) {
}
// deprecated_get_URL.q[i].XMLHttp=0;
delete deprecated_get_URL.i[deprecated_get_URL.q[i].uri];
deprecated_get_URL.q[i] = 0;
}
};
// ↑XMLHttp set ==================
// ---------------------------------------------------------------------//
// @see https://github.com/request/request
var node_url, node_http, node_http2, node_https,
// reuse the sockets (keep-alive connection).
node_http_agent, node_http2_agent, node_https_agent,
//
node_zlib;
/**
* 快速 merge cookie: 只檢查若沒有重複的 key則直接加入。不檢查 path 也不處理 expires, domain,
* secure。<br />
* 為增加效率,不檢查 agent.last_cookie 本身之重複的 cookie。
*
* TODO: create class Cookie, Cookie.prototype.merge(),
* Cookie.prototype.clone()
*
* @param {Object}agent
* node_http_agent / node_https_agent
* @param {Array}cookie
* new cookie to merge
*
* @returns {Object}agent.last_cookie
*
* @inner
*/
function merge_cookie(agent, cookie) {
// 初始化 initialization + 正規化 normalization
var last_cookie = agent.last_cookie;
if (!Array.isArray(last_cookie)) {
last_cookie = agent.last_cookie = agent.last_cookie ? [ agent.last_cookie ]
: [];
}
if (!cookie) {
cookie = [];
} else if (typeof cookie === 'string') {
cookie = cookie.split(';');
}
// assert: Array.isArray(cookie)
// console.log(agent);
// console.log(last_cookie.cookie_hash);
// console.trace(cookie);
// cookie_index_of[key] = index of last_cookie
var cookie_index_of = last_cookie.cookie_index_of;
if (!cookie_index_of) {
if (last_cookie.length > 0) {
// regenerate agent.last_cookie
delete agent.last_cookie;
last_cookie = merge_cookie(agent, last_cookie);
// assert: last_cookie === agent.last_cookie
} else {
last_cookie.cookie_index_of = Object.create(null);
last_cookie.cookie_hash = Object.create(null);
}
cookie_index_of = last_cookie.cookie_index_of;
}
var cookie_hash = last_cookie.cookie_hash;
// assert: !!cookie_hash === true
cookie.forEach(function for_each_cookie_piece(piece) {
piece = piece.trim();
if (!piece)
return;
// [ cookie value without path / domain / expires,
// key, value, extra ]
var matched = piece.match(/^([^=;]+)(?:=([^;]+))?(.*)$/);
library_namespace.debug('last_cookie: ' + last_cookie, 3,
'merge_cookie');
// console.log(matched);
var key, value;
if (matched) {
key = matched[1];
value = matched[2];
} else {
library_namespace.warn([ 'merge_cookie: ', {
// gettext_config:{"id":"invalid-cookie"}
T : 'Invalid cookie?'
}, ' [' + piece + ']' ]);
// treat cookie piece as key
key = piece;
value = '';
}
if (!key)
return;
cookie_hash[key] = value;
if (key in cookie_index_of) {
// assert: (key in cookie_hash) === true
library_namespace.debug([ {
// gettext_config:{"id":"duplicate-cookie-name!-the-later-newcomer-will-prevail"}
T : 'cookie 名稱重複!以後來/新出現者為準。'
}, ' [' + last_cookie[cookie_index_of[key]]
//
+ ']→[' + piece + ']' ], 3, 'merge_cookie');
// remove duplicate cookie: 直接取代。
last_cookie[cookie_index_of[key]] = piece;
} else {
// assert: (key in cookie_hash) === false
library_namespace.debug([ {
T : '正常情況。登記已存在之 cookie。'
} ], 3, 'merge_cookie');
// console.trace(matched);
cookie_index_of[key] = last_cookie.length;
last_cookie.push(piece);
}
});
// console.trace(cookie_hash);
// console.trace(last_cookie);
library_namespace.debug('array: ' + JSON.stringify(last_cookie), 3,
'merge_cookie');
library_namespace.debug('hash: ' + JSON.stringify(cookie_hash), 3,
'merge_cookie');
return last_cookie;
}
_.merge_cookie = merge_cookie;
function set_cookie_to_URL_object(URL_options_to_fetch, agent) {
// console.trace('agent.last_cookie:');
// console.log(agent.last_cookie);
if (agent.last_cookie) {
// 使用 cookie
library_namespace.debug('agent.last_cookie: '
+ JSON.stringify(agent.last_cookie), 3,
'set_cookie_to_URL_object');
library_namespace.debug('agent.last_cookie.cookie_hash: '
+ JSON.stringify(agent.last_cookie.cookie_hash), 3,
'set_cookie_to_URL_object');
var cookie = (URL_options_to_fetch.headers.Cookie ? URL_options_to_fetch.headers.Cookie
+ ';'
: '')
// cookie is Array @ Wikipedia
+ (Array.isArray(agent.last_cookie) ? agent.last_cookie
// 去掉 expires=...; path=/; domain=...; HttpOnly
// 這個動作不做也可以,不影響結果。
.map(function(cookie) {
return cookie.replace(/;.*/, '');
}).join('; ') : agent.last_cookie);
if (cookie) {
URL_options_to_fetch.headers.Cookie = cookie;
} else {
delete URL_options_to_fetch.headers.Cookie;
}
}
library_namespace.debug('Set cookie: '
+ JSON.stringify(URL_options_to_fetch.headers.Cookie), 3,
'set_cookie_to_URL_object');
library_namespace.debug('Set protocol: '
+ URL_options_to_fetch.protocol, 3, 'set_cookie_to_URL_object');
library_namespace.debug('Set headers: '
+ JSON.stringify(URL_options_to_fetch.headers), 3,
'set_cookie_to_URL_object');
}
// ---------------------------------------------------------------------//
// 正處理中之 connections
var get_URL_node_connections = 0,
// 所有 requests
get_URL_node_requests = 0;
// 強制使用POST傳送。
var FORCE_POST = {
FORCE_POST : true
};
var ERROR_BAD_STSTUS = 'BAD STATUS';
var KEY_not_native = library_namespace.env.not_native_keyword;
var has_native_URL = typeof URL === "function" && !URL[KEY_not_native];
/**
* 讀取 URL via node http/https。<br />
* assert: arguments 必須與 get_URL() 相容!
*
* @param {String|Object}URL_to_fetch
* 欲請求之目的 URL or options
* @param {Function}[onload]
* callback when successful loaded. For failure handling, using
* options.onfail(error);
* @param {String}[charset]
* character encoding of HTML web page. e.g., 'UTF-8', big5,
* euc-jp,..
* @param {String|Object}[post_data]
* text data to send when method is POST
* @param {Object}[options]
* 附加參數/設定選擇性/特殊功能與選項
*
* @see https://nodejs.org/api/http.html#http_http_request_options_callback
* https://nodejs.org/api/https.html#https_https_request_options_callback
*
* @since 2015/1/13 23:23:38
*/
function get_URL_node(URL_to_fetch, onload, charset, post_data, options) {
if (!URL_to_fetch) {
onload(undefined, new SyntaxError('No URL input.'));
return;
}
get_URL_node_requests++;
if (get_URL_node_connections >= get_URL_node.connects_limit) {
library_namespace.debug({
// gettext_config:{"id":"waiting-$1-$2-connections-$3"}
T : [ 'Waiting %1/%2 {{PLURAL:%1|connection|connections}}: %3',
// 避免同時開過多 connections 的機制。
get_URL_node_connections, get_URL_node_requests,
String(URL_to_fetch) ]
}, 3, 'get_URL_node');
var _arguments = arguments;
setTimeout(function() {
get_URL_node_requests--;
get_URL_node.apply(null, _arguments);
}, 500);
return;
}
// 進入 request 程序
get_URL_node_connections++;
// 前導作業。
if (library_namespace.is_Object(charset)) {
post_data = charset;
charset = null;
}
// 正規化並提供可隨意改變的同內容參數,以避免修改或覆蓋附加參數。
options = library_namespace.new_options(options);
// console.log('-'.repeat(79));
// console.log(JSON.stringify(options));
// console.log(options.form_data);
if (options.form_data && options.form_data !== to_form_data_generated) {
// console.trace(options);
// TODO: charset for post_data
to_form_data(post_data, function(data) {
// console.log(data.toString().slice(0,800));
// console.log('>> ' + data.toString().slice(-200));
options.form_data = to_form_data_generated;
get_URL_node(URL_to_fetch, onload, charset, data, options);
}, options.form_data);
return;
}
// ------------------------------------------------------
if (library_namespace.is_Object(URL_to_fetch) && URL_to_fetch[KEY_URL]) {
Object.assign(options, URL_to_fetch);
// 注意: options.onload 另有用途!
// https://xhr.spec.whatwg.org/#handler-xhr-onloadstart
// onload = options.onload || onload;
post_data = options.post || post_data;
charset = options.charset || charset;
URL_to_fetch = options[KEY_URL];
}
// 不改變 options。
var agent = options.agent;
if (typeof URL_to_fetch === 'string' && URL_to_fetch.startsWith('//')) {
// 處理 '//domain.org/path' 的情況。
URL_to_fetch = (agent && agent.protocol || 'https:') + URL_to_fetch;
}
var URL_options_to_fetch = normalize_URL_to_fetch(URL_to_fetch,
charset, options);
// assert: library_namespace.is_URI(URL_options_to_fetch)
// console.trace([ URL_to_fetch, URL_options_to_fetch ]);
if (typeof onload === 'object') {
library_namespace.debug(
'Trying to JSONP, insert page, need callback.', 3,
'get_URL_node');
// library_namespace.run(URL_options_to_fetch);
for ( var callback_param in onload) {
if (callback_param
&& typeof onload[callback_param] === 'function') {
// 模擬 callback。
// callback_param: callback parameter
URL_options_to_fetch.search_params[callback_param] = 'cb';
onload = onload[callback_param];
break;
}
}
}
// assert: 自此開始不會改變 URL也不會中途 exit 本函數。
if (post_data && !options.form_data) {
if (library_namespace.is_Object(post_data)
&& options.headers
&& typeof options.headers['Content-Type'] === 'string'
&& options.headers['Content-Type']
.includes('application/json')) {
post_data = JSON.stringify(post_data) || FORCE_POST;
} else {
post_data = library_namespace.Search_parameters(post_data)
.toString(charset)
|| FORCE_POST;
}
}
if (!onload && typeof options.onchange === 'function') {
onload = function() {
options.onchange(readyState_done);
};
}
if (options.async === false && onload || typeof onload !== 'function') {
onload = false;
}
// console.trace(URL_options_to_fetch);
// node_http.request(options): options needs a .path
// https://nodejs.org/dist/latest/docs/api/http.html#http_http_request_options_callback
URL_options_to_fetch.path = URL_options_to_fetch.pathname
+ URL_options_to_fetch.search;
var URL_is_https = /^https:?$/i.test(URL_options_to_fetch.protocol);
// console.trace([ URL_to_fetch, URL_options_to_fetch.toString() ]);
URL_to_fetch = URL_options_to_fetch.toString();
// assert: {String}URL_to_fetch,
// library_namespace.is_URI(URL_options_to_fetch)
/**
* https://stackoverflow.com/questions/53593182/client-network-socket-disconnected-before-secure-tls-connection-was-established
* mh160.js 必須使用 requesthttps-proxy-agent 才能正常工作 TODO:
* socks-proxy-agent
*
* <code>
// https://techcult.com/free-proxy-software-for-windows-10/#1_Ultrasurf
> SET http_proxy=http://127.0.0.1:9666
> node u17.js 镇魂街
</code>
*/
/**
* <code>
// http://anonproxyserver.sourceforge.net/
// https://www.proxynova.com/proxy-server-list/country-tw/
var http = require("http");
var options = {
host: "211.22.233.69",
port: 3128,
path: "http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout",
//method: 'GET',
headers: {
Host: "dict.revised.moe.edu.tw"
}
};
var request=http.request(options, function(response) {
console.log(response.statusCode);
console.log(response.headers);
var data = [], length = 0;
response.on('data', function(chunk) {length += chunk.length;data.push(chunk);});
response.on('end', function() {data = Buffer.concat(data, length);console.log(data+'')});
});
request.end();
require('./work_crawler_loader.js'); var PROXY='localhost:8080';
CeL.get_URL('https://zh.wikipedia.org/wiki/Special:%E6%9C%80%E8%BF%91%E6%9B%B4%E6%94%B9',function(X){console.log(X.responseText)},null,null,{proxy:PROXY});
CeL.get_URL('https://zh.wikipedia.org/wiki/Special:%E6%9C%80%E8%BF%91%E6%9B%B4%E6%94%B9',function(X){console.log(X.responseText)});
CeL.get_URL('http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout',function(X){console.log(X.responseText)},null,null,{proxy:PROXY});
CeL.get_URL('http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout',function(X){console.log(X.responseText)});
// TODO: test agent, cookie
</code>
*/
var proxy_original_agent,
// using_proxy_server
proxy_server = options.proxy
// https://curl.haxx.se/docs/manpage.html
// https://superuser.com/questions/876100/https-proxy-vs-https-proxy
// https://docs.oracle.com/cd/E56344_01/html/E54018/gmgas.html
// https://stackoverflow.com/questions/32824819/difference-between-http-proxy-https-proxy-and-proxy
|| URL_is_https && process.env.HTTPS_PROXY
// `SET http_proxy=http://127.0.0.1:8080`
|| process.env.http_proxy;
if (!proxy_server
|| !(proxy_server = library_namespace.URI(proxy_server))) {
;
} else if (URL_is_https) {
library_namespace.debug({
// gettext_config:{"id":"using-https-proxy-to-get-url-$1"}
T : [ 'Using HTTPS proxy to get URL: %1', URL_to_fetch ]
}, 2, 'get_URL_node');
// ... just add the special agent:
proxy_original_agent = proxy_server.agent = agent;
agent = new HttpsProxyAgent(proxy_server);
// 複製必要的舊屬性。
if (proxy_original_agent && proxy_original_agent.last_cookie) {
// 複製原agent的cookie設定。 @see merge_cookie()
agent.last_cookie = proxy_original_agent.last_cookie;
}
// https://github.com/TooTallNate/node-https-proxy-agent/blob/master/index.js
// ALPN is supported by Node.js >= v5.
// attempt to negotiate http/1.1 for proxy servers that support
// http/2
if (!('ALPNProtocols' in URL_options_to_fetch)) {
URL_options_to_fetch.ALPNProtocols = [ 'http 1.1' ];
}
} else {
library_namespace.debug({
// gettext_config:{"id":"using-http-proxy-to-get-url-$1"}
T : [ 'Using HTTP proxy to get URL: %1', URL_to_fetch ]
}, 2, 'get_URL_node');
// https://www.proxynova.com/proxy-server-list/country-tw/
// proxy_server.URL_to_fetch = URL_to_fetch;
// 代理伺服器 using proxy server
// https://stackoverflow.com/questions/3862813/how-can-i-use-an-http-proxy-with-node-js-http-client
// https://cnodejs.org/topic/530f41e75adfcd9c0f1c8c16
URL_options_to_fetch = get_proxy_URL(proxy_server,
URL_options_to_fetch, URL_to_fetch);
}
if (!URL_options_to_fetch.protocol) {
URL_options_to_fetch.protocol = agent && agent.protocol
// 直接設定。 default: https://
|| (proxy_server ? 'http:' : 'https:');
URL_is_https = URL_options_to_fetch.protocol === 'https:';
}
if (agent) {
library_namespace.debug({
// gettext_config:{"id":"using-new-agent"}
T : agent === true ? '使用新 agent。'
// gettext_config:{"id":"using-custom-agent"}
: '使用自定義 agent。'
}, 6, 'get_URL_node');
if (agent === true) {
// use new agent.
options.agent = agent = URL_is_https ? new node_https.Agent
: new node_http.Agent;
} else if (agent.protocol
// agent.protocol 可能是 undefined。
&& agent.protocol !== URL_options_to_fetch.protocol) {
var message = {
// gettext_config:{"id":"the-custom-agent-is-different-from-the-url-and-will-try-to-adopt-the-conforming-agreement-$1"}
T : [ '自定義 agent 與 URL 之協定不同,將嘗試採用符合的協定:%1',
//
agent.protocol + ' !== ' + URL_options_to_fetch.protocol ]
};
if (options.no_protocol_warn) {
library_namespace.debug(message, 3, 'get_URL_node');
} else {
library_namespace.warn([ 'get_URL_node: ', message ]);
}
// use new agent.
// assert: options.agent === agent
agent = URL_is_https ? new node_https.Agent
: new node_http.Agent;
// 複製必要的舊屬性。
if (options.agent.last_cookie) {
// 複製原agent的cookie設定。 @see merge_cookie()
agent.last_cookie = options.agent.last_cookie;
}
}
} else {
// gettext_config:{"id":"using-generic-agent"}
library_namespace.debug('採用泛用的 agent。', 6, 'get_URL_node');
agent = URL_is_https ? node_https_agent : node_http_agent;
}
// console.log([ options.cookie, agent.last_cookie ]);
if (options.cookie && !agent.last_cookie) {
library_namespace.debug({
// gettext_config:{"id":"reset-the-cookie-to-$1"}
T : [ '重新設定 cookie 成:%1', options.cookie ]
}, 3, 'get_URL_node');
agent.last_cookie = options.cookie;
}
var request, finished,
// result_Object 模擬 XMLHttp response。
result_Object = {
// node_agent : agent,
// .url @ fetch()
// url : URL_to_fetch,
// https://developer.mozilla.org/zh-TW/docs/Web/API/Response
// .useFinalURL @ fetch()
// useFinalURL : URL_to_fetch,
// 因為可能 redirecting 過,這邊列出的才是最終 URL。
// https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/responseURL
responseURL : URL_to_fetch
},
// assert: 必定從 _onfail 或 _onload 作結,以確保會註銷登記。
// 本函數unregister()應該放在所有本執行緒會執行到onload的程式碼中。
unregister = function() {
/**
* @see http://stackoverflow.com/questions/24667122/http-request-timeout-callback-in-node-js
*
* sometimes both timeout callback and error callback will be called
* (the error inside the error callback is ECONNRESET - connection
* reset)
*
* there is a possibilities that it fires on('response',
* function(response)) callback altogether
*/
if (finished) {
return true;
}
// 註銷登記。
finished = true;
get_URL_node_requests--;
get_URL_node_connections--;
if (timeout_id) {
library_namespace.debug('clear timeout '
+ time_message(timeout) + ' [' + URL_to_fetch + ']', 3,
'get_URL_node');
// console.trace('clear timeout ' + URL_to_fetch);
clearTimeout(timeout_id);
}
},
// on failed
_onfail = function(error) {
if (unregister()) {
// console.log('exit: 預防 timeout 時重複執行。');
return;
}
// console.trace([ options.error_count, options.error_retry ]);
// 連線逾期/失敗時再重新取得頁面之重試次數。
if (options.error_retry >= 1
// 例如當遇到 404 或 502 時,再多嘗試一下。
&& !(options.error_retry <= options.error_count)) {
if (!options.get_URL_cloned) {
// 不動到原來的 options。
options = Object.clone(options);
options.get_URL_cloned = true;
}
if (options.error_count >= 1) {
options.error_count++;
} else {
options.error_count = 1;
}
options[KEY_URL] = URL_to_fetch;
// Failed to get [' + URL_to_fetch + '].
library_namespace.log([ 'get_URL_node: ', {
// gettext_config:{"id":"retry-$1-$2-$3"}
T : [ 'Retry %1/%2: %3', options.error_count,
//
options.error_retry, localize_error(error) ]
} ]);
// console.error(error);
// library_namespace.set_debug(3);
get_URL_node(options, onload, charset, post_data);
// console.trace(options);
return;
}
if (typeof options.onfail === 'function') {
options.onfail.call(result_Object, error);
return;
}
if (!options.no_warning
// 應已在 _ontimeout 出過警告訊息。
&& error.code !== 'TIMEOUT') {
if (error.code === 'ENOTFOUND') {
library_namespace.error([ 'get_URL_node: ', {
// gettext_config:{"id":"url-not-found-$1"}
T : [ 'URL not found: [%1]', URL_to_fetch ]
} ]);
} else if (error.code === 'EPROTO'
&& require('tls').DEFAULT_MIN_VERSION === 'TLSv1.2'
&& library_namespace.platform('node', 12)) {
library_namespace.error([ 'get_URL_node: ', {
T :
// gettext_config:{"id":"node.js-v12-and-later-versions-disable-tls-v1.0-and-v1.1-by-default"}
'Node.js v12 and later versions disable TLS v1.0 and v1.1 by default.'
}, {
T :
// gettext_config:{"id":"please-set-tls.default_min_version-=-tlsv1-first"}
'Please set tls.DEFAULT_MIN_VERSION = "TLSv1" first!'
}, ' [' + URL_to_fetch + ']' ]);
/**
* <code>
To solve:
get_URL_node: Retry 1/4: Error: write EPROTO 14180:error:1425F102:SSL routines:ssl_choose_client_version:unsupported protocol:c:\ws\deps\openssl\openssl\ssl\statem\statem_lib.c:1922:
require('tls').DEFAULT_MIN_VERSION = 'TLSv1';
</code>
*/
} else {
library_namespace.error([ 'get_URL_node: ', {
// gettext_config:{"id":"got-error-when-retrieving-$1-$2"}
T : [ 'Got error when retrieving [%1]: %2',
//
URL_to_fetch, localize_error(error) ]
} ]);
// 這裡用太多並列處理,會造成 error.code "EMFILE"。
// console.error(error);
// console.error(options);
}
}
// 在出現錯誤時,將 onload 當作 callback。並要確保 {Object}response
// 因此應該要先檢查 error 再處理 response
typeof onload === 'function' && onload(result_Object, error);
},
// on success
_onload = function(response) {
// response object: Class: http.IncomingMessage
// 在這邊不過剛開始從伺服器得到資料因此還不可執行unregister()否則依然可能遇到timeout。
if (finished) {
return;
}
if (options.onresponse) {
options.onresponse(response);
}
// https://xhr.spec.whatwg.org/#handler-xhr-onloadstart
if (false && options.onloadstart) {
options.onloadstart();
}
// 若原先有agent應該合併到原先的agent而非可能為暫時性/泛用的agent。
merge_cookie(options.agent || agent, response.headers['set-cookie']);
// 先合併完cookie之後才能轉址否則會漏失掉須設定的cookie。
if (false && response.complete
&& (response.statusCode / 100 | 0) !== 2) {
console.log('response:');
console.log(response);
}
if ((response.statusCode / 100 | 0) === 3
&& response.headers.location
&& response.headers.location !== URL_to_fetch
&& !options.no_redirect) {
if (unregister()) {
// 預防 timeout 時重複執行。
return;
}
try {
// request.abort();
request.destroy();
} catch (e) {
}
// e.g., 301
if (!options.get_URL_cloned) {
// 不動到原來的 options。
options = Object.clone(options);
options.get_URL_cloned = true;
}
options[KEY_URL] = new URL(response.headers.location,
URL_to_fetch);
library_namespace.debug({
// gettext_config:{"id":"$1-redirecting-to-$2-←-$3"}
T : [ '%1 Redirecting to [%2] ← [%3]', response.statusCode,
options[KEY_URL], URL_to_fetch ]
}, 1, 'get_URL_node');
get_URL_node(options, onload, charset,
// 重新導向的時候亦傳送 post data。For wiki API
// e.g., https://github.com/kanasimi/CeJS/issues/12
// 但piaotian.js搜索作品名稱直接302導向時加入post_data會產生 405 Not Allowed
response.statusCode != 302 && post_data);
return;
}
library_namespace.debug({
// gettext_config:{"id":"response-headers-$1"}
T : [ 'response HEADERS: %1',
//
JSON.stringify(response.headers) ]
}, 4, 'get_URL_node._onload');
// 模擬 Response of fetch()
// https://developer.mozilla.org/zh-TW/docs/Web/API/Response
Object.assign(result_Object, {
redirected : !!options.get_URL_cloned,
// {Number}response.statusCode
// https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/status
status : response.statusCode,
statusText : response.statusMessage,
// XMLHttp.headers['content-type']==='text/html; charset=utf-8'
headers : response.headers
});
// 在有 options.onfail 時僅 .debug()。但這並沒啥條理...
if (options.onfail || (response.statusCode / 100 | 0) === 2) {
library_namespace.debug({
// gettext_config:{"id":"http-status-code-$1-$2"}
T : [ 'HTTP status code: %1 %2', response.statusCode,
URL_to_fetch ]
}, 2, 'get_URL_node');
} else if (!options.no_warning) {
library_namespace.warn([ 'get_URL_node: ', {
// gettext_config:{"id":"exception-http-status-code-$1-$2"}
T : [ 'Exception HTTP status code %1: %2',
//
response.statusCode, URL_to_fetch ]
} ]);
// console.trace(response);
}
// node.js會自動把headers轉成小寫。
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
// 在503之類的情況下。可能沒"Content-Type:"。這時 response 將無.type。
if (response.headers['content-type']) {
// MIME type, media-type: XMLHttp.type
result_Object.type = response.headers['content-type']
// charset: XMLHttp.charset
.replace(/;(.*)$/, function($0, $1) {
var matched = $1.match(/[; ]charset=([^;]+)/i);
if (matched) {
result_Object.charset = matched[1].trim();
}
return '';
}).trim();
}
if (response.headers['content-disposition']) {
// 從 Content-Disposition 中抽取出檔名。
// ext-value = charset "'" [ language ] "'" value-chars
var matched = response.headers['content-disposition']
.match(/ filename\*\s*=\s*([^';]+)'([^';]*)'([^';]+)/);
if (matched) {
matched = matched[3];
} else if (matched = response.headers['content-disposition']
.match(/ filename\s*=\s*([^';]+)/)) {
matched = matched[1];
}
if (matched && (matched = matched.trim())) {
matched = (matched.match(/^"(.*)"$/)
|| matched.match(/^'(.*)'$/) || [ , matched ])[1];
if (false) {
console.log([ matched, result_Object.charset, charset,
!!library_namespace.decode_URI_component ]);
}
try {
if (!/%[\da-f]{2}/.test(matched)) {
// 有一些網站經過這個轉換似乎就能夠獲得正確的檔案名稱。
matched = escape(matched);
}
if (false && (result_Object.charset || charset)
&& library_namespace.decode_URI_component) {
// 現在沒有實例需要用到這個部分。
matched = library_namespace.decode_URI_component(
matched, result_Object.charset || charset);
} else {
matched = decodeURIComponent(matched);
// 有的時候還需要這一項。
// matched = unescape(matched);
}
library_namespace.debug({
// gettext_config:{"id":"file-name-$1"}
T : [ '檔案名稱:%1', matched ]
}, 3, 'get_URL_node');
} catch (e) {
// TODO: handle exception
}
result_Object.filename = library_namespace
.to_file_name(matched);
}
}
// listener must be a function
if (typeof onload !== 'function'
//
&& !options.write_to && !options.write_to_directory) {
// 照理unregister()應該放這邊,但如此速度過慢。因此改放在 _onload 一開始。
unregister();
library_namespace.warn([ 'get_URL_node: ', {
// gettext_config:{"id":"got-url-$1-but-there-is-no-listener"}
T : [ 'Got URL [%1], but there is no listener!',
//
URL_to_fetch ]
} ]);
// console.log(response);
return;
}
library_namespace.debug({
// gettext_config:{"id":"waiting-to-receive-data-transferred-back-from-url-$1"}
T : [ '等待接收從網址 [%1] 傳輸回的資料……', URL_to_fetch ]
}, 3, 'get_URL_node');
var flow_encoding = response.headers['content-encoding'];
flow_encoding = flow_encoding && flow_encoding.trim().toLowerCase();
if (false) {
var pipe = response;
if (flow_encoding === 'gzip')
pipe = pipe.pipe(node_zlib.createGunzip());
else if (flow_encoding === 'deflate')
pipe = pipe.pipe(node_zlib.createInflate());
pipe = pipe.pipe(node_fs.createWriteStream(file_path));
}
// 準備開始接收資料
// options.ondatastart: 非正規標準
if (options.ondatastart) {
options.ondatastart(response);
}
// options.onload https://xhr.spec.whatwg.org/#handler-xhr-onload
/** {Array} [ {Buffer}, {Buffer}, ... ] */
var data = [], length = 0,
// total_size https://xhr.spec.whatwg.org/#progressevent
total_length = +response.headers['content-length'], lengthComputable = total_length >= 0;
response.on('data', function(chunk) {
// {Buffer}chunk
length += chunk.length;
var message = [ (options.write_to ? options.write_to + ': '
: '')
// + chunk.length + '/'
+ length
+ (total_length ? '/' + total_length : '')
+ ' bytes ('
// 00% of 0.00MiB
+ (total_length ? (100 * length / total_length | 0)
+ '%, ' : '')
+ (length / 1.024 / (/* Date.now() */(new Date)
.getTime() - start_time)).toFixed(2)
+ ' KiB/s)' ];
message.push(': ' + URL_to_fetch);
library_namespace.debug('receive BODY.length: '
+ message.join(''), 4, 'get_URL_node');
if (options.show_progress && length !== total_length) {
if (!(options.show_progress > 1)) {
message.pop();
}
library_namespace.log_temporary(message.join(''));
}
if (length > options.MAX_BUFFER_SIZE) {
if (data)
data = null;
} else {
data.push(chunk);
}
// node_fs.appendFileSync('get_URL_node.data', chunk);
if (options.ondata) {
// 注意: 這邊的 chunk 可能是 gzip 之後的資料!
options.ondata(chunk);
}
// https://xhr.spec.whatwg.org/#handler-xhr-onprogress
if (false && options.onprogress) {
options.onprogress(lengthComputable, length, total_length);
}
});
// https://iojs.org/api/http.html#http_http_request_options_callback
response.on('end', function() {
library_namespace.debug('end(): [' + response.statusCode + '] '
+ URL_to_fetch, 2, 'get_URL_node');
// 照理應該放這邊,但如此速度過慢。因此改放在 _onload 一開始。
// unregister();
options.onend && options.onend();
// console.log('No more data in response: ' + URL_to_fetch);
// it is faster to provide the length explicitly.
data = data && Buffer.concat(data, length);
// console.log(data.slice(0, 200));
// console.log(data.slice(0, 200).toString());
if (proxy_original_agent) {
// recover properties
proxy_original_agent.last_cookie = agent.last_cookie;
}
// 基本檢測。
if ((response.statusCode / 100 | 0) !== 2
// 例如當遇到 404 或 502 時,再多嘗試一下。
&& options.error_retry >= 1
// 本條件參考 _onfail。
&& !(options.error_retry <= options.error_count)) {
_onfail(ERROR_BAD_STSTUS);
return;
}
if (!data) {
;
} else if (options.verify) {
// test: invalid content type
if (typeof options.verify === 'function') {
if (!options.verify(data)) {
_onfail('INVALID');
return;
}
} else {
// assert: CeL.application.storage.file included
// e.g., options.verify === 'png'
var file_type = library_namespace.file_type(data,
options.verify);
if (file_type.verified === false) {
_onfail('Invalid ' + options.verify);
return;
}
}
} else if (data.length === 0) {
// 若是容許空內容,應該特別指定 options.allow_empty。
if (!options.allow_empty) {
_onfail('EMPTY');
return;
}
}
// https://github.com/nodejs/node/blob/master/doc/api/zlib.md#compressing-http-requests-and-responses
// https://nodejs.org/docs/latest/api/zlib.html
// https://gist.github.com/narqo/5265413
// https://github.com/request/request/blob/master/request.js
// http://stackoverflow.com/questions/8880741/node-js-easy-http-requests-with-gzip-deflate-compression
// http://nickfishman.com/post/49533681471/nodejs-http-requests-with-gzip-deflate-compression
if (flow_encoding && data) {
library_namespace.debug('content-encoding: '
+ flow_encoding, 5, 'get_URL_node');
switch (flow_encoding) {
case 'gzip':
library_namespace.debug('gunzip ' + data.length
+ ' bytes data...', 2, 'get_URL_node');
/**
* <code>
可能因為呼叫到舊版library於此有時會出現 "TypeError: Object #<Object> has no method 'gunzipSync'"
有時會有 Error: unexpected end of file
</code>
*/
try {
data = node_zlib.gunzipSync(data);
} catch (error) {
library_namespace.error(
// get_URL_node: Error: node_zlib.gunzipSync():
// Error: unexpected end of file [http://...]
'get_URL_node: Error: node_zlib.gunzipSync(): '
+ localize_error(error) + ' ['
+ URL_to_fetch + ']');
if (false) {
console.log(error);
console.log(URL_options_to_fetch);
console.log(node_zlib);
console.log(data);
console.trace(
//
'get_URL_node: Error: node_zlib.gunzipSync()');
console.error(error.stack);
}
// Release memory. 釋放被占用的記憶體.
data = null;
_onfail(error);
return;
}
break;
case 'deflate':
library_namespace.debug('deflate data ' + data.length
+ ' bytes...', 2, 'get_URL_node');
data = node_zlib.deflateSync(data);
break;
case 'br':
library_namespace.debug('brotli data ' + data.length
+ ' bytes...', 2, 'get_URL_node');
data = node_zlib.brotliDecompressSync(data);
break;
default:
library_namespace.warn([ 'get_URL_node: ', {
// gettext_config:{"id":"unknown-http-compression-method-$1"}
T : [ 'Unknown HTTP compression method: [%1]',
//
flow_encoding ]
} ]);
break;
}
}
// ------------------------------
if (data && response.statusCode === 403
//
&& data.toString().includes('Cloudflare')
//
&& data.toString().includes(' id="captcha-bypass"')) {
// console.log(data.toString());
library_namespace.error([ 'get_URL_node: ', {
// https://github.com/Anorov/cloudflare-scrape
T : // TODO: https://github.com/codemanki/cloudscraper
// gettext_config:{"id":"you-need-to-bypass-the-ddos-protection-by-cloudflare"}
'You need to bypass the DDoS protection by Cloudflare!'
} ]);
}
// ------------------------------
// setup data of result_Object
result_Object.buffer = data;
// https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/response
result_Object.response = data;
// non-standard 非標準: 設定 charset = 'buffer' 的話,將回傳 {Buffer}。
if (data && charset !== 'buffer') {
// 未設定 charset 的話default charset: UTF-8.
// buffer.toString(null) will throw!
data = data.toString(charset || undefined/* || 'utf8' */);
}
if (library_namespace.is_debug(4)) {
library_namespace.debug(
//
'BODY: ' + data, 1, 'get_URL_node');
}
// result_Object模擬 XMLHttp。
result_Object.responseText = data;
// ------------------------------
if ((response.statusCode / 100 | 0) !== 2) {
// console.trace(data);
// ssert: options.error_retry >= 1 ? 最後一次 error
// : BAD STATUS and get something in `this.response`
_onfail(ERROR_BAD_STSTUS);
return;
}
if (unregister()) {
// 預防 timeout 時重複執行。
return;
}
// assert: 執行至此表示成功取得資料、沒有錯誤,
// 開始正常運作至結尾,不會再有中途 return。
// TODO: 確保資料完整,例如檢查結尾碼。
// .save_to
if (data && (options.write_to || options.write_to_directory)) {
var file_path = options.write_to
// save to: 設定寫入目標。
|| (options.write_to_directory
//
+ library_namespace.env.path_separator
//
+ library_namespace.to_file_name(
//
URL_to_fetch.replace(/#.*/, '').replace(
/[\\\/:*?"<>|]/g, '_')))
// 避免 Error: ENAMETOOLONG: name too long
.slice(0, 256);
if (!options.no_warning) {
library_namespace.info([ 'get_URL_node: ', {
T : [
// gettext_config:{"id":"write-$2-bytes-to-file-$1-$3"}
'Write %2 {{PLURAL:%2|byte|bytes}} to file [%1]: %3'
//
, file_path, data.length, URL_to_fetch ]
} ]);
}
try {
var fd = node_fs.openSync(file_path, 'w');
// TODO: use response.pipe(write_stream);
// @see GitHub.updater.node.js
node_fs.writeSync(fd, data, 0, data.length, null);
node_fs.closeSync(fd);
// set file modify date
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
// https://tools.ietf.org/html/rfc7231#section-7.1.1.2
if (response.headers['date']) {
try {
// The "Date" header field represents the date
// and time at which the message was originated
// fs.utimesSync(path, atime, mtime)
// atime: the last time this file was accessed
node_fs.utimesSync(file_path, new Date,
// mtime: the last time this file was modified
response.headers['date']);
} catch (e) {
// TODO: handle exception
}
}
} catch (e) {
library_namespace.error([ 'get_URL_node: ', {
T : [
// gettext_config:{"id":"failed-to-write-$2-bytes-to-$1-$3"}
'Failed to write %2 {{PLURAL:%2|byte|bytes}} to [%1]: %3'
//
, file_path, data.length, URL_to_fetch ]
} ]);
console.error(e);
}
}
if (typeof options.content_processor === 'function') {
options.content_processor(
// ({Buffer}contains, URL, status)
data, URL_to_fetch, response.statusCode);
}
if (typeof options.check_reget === 'function'
// check_reget(XMLHttp)
&& options.check_reget(result_Object, options)) {
options[KEY_URL] = URL_to_fetch;
get_URL_node(options, onload, charset, post_data);
return;
}
if (typeof onload === 'function') {
onload(result_Object, !data && !options.allow_empty);
}
// Release memory. 釋放被占用的記憶體.
data = null;
// node_fs.appendFileSync('get_URL_node.data', '\n');
});
};
// --------------------------------------------------------------------
// console.trace([ options.headers, URL_options_to_fetch.headers ]);
// https://fetch.spec.whatwg.org/#forbidden-header-name
// 必要的 headers: User-Agent, Accept-Language。其他是為了模擬得更真實點。
URL_options_to_fetch.headers = Object.assign({
// 網站的主機名稱。
Host : URL_options_to_fetch.host,
// User Agent
'User-Agent' : get_URL_node.default_user_agent,
// https://developer.mozilla.org/zh-CN/docs/Glossary/Quality_values
Accept : 'text/html,application/xhtml+xml,application/xml;q=0.9'
// 少了 '*/*' CrossRef API 會回應 406 "No acceptable resource available."
+ ',image/avif' + ',image/webp,image/apng,*/*;q=0.8'
+ ',application/signed-exchange;v=b3;q=0.9',
// Accept : 'application/json, text/plain, */*',
// 為了防止 Cloudflare bot protection(?) 阻擋,必須加上 Accept-Language。
// TODO: get language from system infomation
// dm5.js using "Microsoft-IIS/8.5",
// needs 'Accept-Language' to search!
'Accept-Language' : 'zh-TW,zh;q=0.9' + ',ja;q=0.8' + ',en;q=0.7',
// DNT : 1,
Connection : 'keep-alive',
'Upgrade-Insecure-Requests' : 1,
// 'sec-ch-ua-mobile' : '?0',
// 'sec-ch-ua-platform' : 'Windows',
// TE 請求型頭部用來指定用戶代理希望使用的傳輸編碼類型。
// 可以將其非正式稱為 Accept-Transfer-Encoding這個名稱顯得更直觀一些。
// 當 TE 設置為 trailers 時,如果服務端支持並且返回了
// Transfer-Encoding: trailers那麼同時也必須返回另一個響應標頭 Trailer
// TE : 'Trailers',
// Origin : URL_options_to_fetch.protocol + '://' +
// URL_options_to_fetch.host
Pragma : 'no-cache',
// 'max-age=0'
'Cache-Control' : 'no-cache'
}, options.fetch_type === 'image' ? {
// /\.(jpg|png|webp)$/i.test(URL_to_fetch.replace(/[?#].*/, ''))
Accept :
// 每次請求重設這些標頭。
'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
'Sec-Fetch-Dest' : /* options.fetch_type || */'image',
'Sec-Fetch-Mode' : 'no-cors',
'Sec-Fetch-Site' : 'cross-site',
'Sec-Fetch-User' : undefined
} : {
// 為了順暢使用 Cloudflare必須加上 Sec-Fetch-headers e.g., mymhh.js
// https://blog.kalan.dev/fetch-metadata-request-headers/
'Sec-Fetch-Dest' : options.fetch_type || 'document',
'Sec-Fetch-Mode' : 'navigate',
'Sec-Fetch-Site' : 'none',
'Sec-Fetch-User' : '?1',
}, options.headers, URL_options_to_fetch.headers);
for ( var key in URL_options_to_fetch.headers) {
if (URL_options_to_fetch.headers[key] === undefined)
delete URL_options_to_fetch.headers[key];
}
// delete URL_options_to_fetch.headers.Referer;
// console.log(options.headers);
if (false && /Chrome\//.test(get_URL_node.default_user_agent)) {
Object.assign(URL_options_to_fetch.headers, {
'sec-ch-ua' :
//
'" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"'
});
}
if (node_zlib.gunzipSync
// && node_zlib.deflateSync
) {
// 早期 node v0.10.25 無 zlib.gunzipSync。Added in: v0.11.12
// 'gzip, deflate, *'
URL_options_to_fetch.headers['Accept-Encoding'] = 'gzip, deflate'
// Added in: v11.7.0, v10.16.0
+ (node_zlib.brotliDecompressSync ? ', br' : '');
}
// console.trace(URL_options_to_fetch.headers);
if (false) {
// @see jQuery
if (!options.crossDomain
&& !URL_options_to_fetch.headers["X-Requested-With"]) {
URL_options_to_fetch.headers["X-Requested-With"] = "XMLHttpRequest";
}
}
if (post_data) {
URL_options_to_fetch.method = 'POST';
var _post_data = post_data === FORCE_POST ? '' : post_data;
if (false && options.form_data) {
// console.log('-'.repeat(79));
// console.log(_post_data.to_Array().content_length);
// console.log(_post_data);
}
Object.assign(URL_options_to_fetch.headers, {
'Content-Type' : options.headers
//
&& options.headers['Content-Type']
//
|| (options.form_data ? 'multipart/form-data; boundary='
// boundary 存入→ post_data.boundary
+ _post_data.boundary : 'application/x-www-form-urlencoded'),
// prevent HTTP 411 錯誤 需要內容長度頭 (411 Length Required)
'Content-Length' : options.form_data
//
? _post_data.to_Array().content_length
// NG: _post_data.length
: charset ? Buffer.byteLength(_post_data, charset) : Buffer
.byteLength(_post_data)
});
}
if (options.method) {
// e.g., 'HEAD'
URL_options_to_fetch.method = options.method;
}
URL_options_to_fetch.agent = agent;
set_cookie_to_URL_object(URL_options_to_fetch, agent);
if (library_namespace.is_debug(3)) {
library_namespace.debug('Set headers: '
+ JSON.stringify(URL_options_to_fetch.headers), 3,
'get_URL_node');
console.log(URL_options_to_fetch.headers);
}
// console.log(URL_options_to_fetch);
try {
// console.trace([ URL_to_fetch, URL_options_to_fetch ]);
// request scheme
request = URL_is_https ? node_https : node_http;
// from node.js 10.9.0
// http.request(url[, options][, callback])
// request: Class: http.ClientRequest
request = request.request(URL_options_to_fetch, _onload);
} catch (e) {
// e.g., _http_client.js:52
if (false) {
throw new TypeError(
// gettext_config:{"id":"request-path-contains-unescaped-characters"}
'Request path contains unescaped characters');
}
_onfail(e);
return;
}
if (options.max_listeners >= 0) {
/**
* 最多平行取得檔案的數量。 <code>
incase "MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 connect listeners added. Use emitter.setMaxListeners() to increase limit"
</code>
*/
request.setMaxListeners(options.max_listeners);
}
if (post_data) {
var _post_data = post_data === FORCE_POST ? '' : post_data;
// console.log(_post_data);
if (options.form_data) {
(function write_to_request(data) {
if (Array.isArray(data)) {
data.forEach(function(chunk) {
write_to_request(chunk);
})
} else {
request.write(data);
}
})(_post_data.to_Array());
} else if (typeof _post_data === 'string') {
library_namespace.debug('set post data: length '
+ _post_data.length, 3, 'get_URL_node');
if (_post_data) {
library_namespace.debug('set post data: '
+ (_post_data.length <= 800
//
|| library_namespace.is_debug(6) ? _post_data
: _post_data.slice(0, 800) + '...'), 3,
'get_URL_node');
}
request.write(_post_data);
} else {
library_namespace.error({
// gettext_config:{"id":"invalid-post-data-$1"}
T : [ 'Invalid POST data: %1', JSON.stringify(post_data) ]
});
}
}
/** {Natural}timeout in ms for get URL. 逾時ms數 */
var timeout = options.timeout || get_URL_node.default_timeout, timeout_id,
//
_ontimeout = function(e) {
// 可能已被註銷。
if (finished) {
return;
}
try {
// http://hylom.net/node-http-request-get-and-timeout
// timeoutイベントは発生しているものの、イベント発生後も引き続きレスポンスを待ち続けている
// request.end();
// request.abort();
request.destroy();
} catch (err) {
// TODO: handle exception
}
if (!options.no_warning) {
library_namespace.info([ 'get_URL_node: ', {
// gettext_config:{"id":"connection-timeout-for-$1-$2"}
T : [ 'Connection timeout for %1: [%2]',
//
time_message(timeout), URL_to_fetch ]
} ]);
}
if (!e) {
e = new Error('Timeout ' + time_message(timeout) + ': ['
+ URL_to_fetch + ']');
e.code = 'TIMEOUT';
}
_onfail(e);
};
if (timeout > 0) {
// setTimeout method 1
// 此方法似乎不能確實於時間到時截斷。或許因為正在 handshaking?
request.setTimeout(timeout);
// https://nodejs.org/api/http.html#http_request_settimeout_timeout_callback
// http://stackoverflow.com/questions/14727115/whats-the-difference-between-req-settimeout-socket-settimeout
request.on('timeout', _ontimeout);
// setTimeout method 2
// {Object}timeout_id @ node.js
timeout_id = setTimeout(_ontimeout, timeout);
library_namespace.debug({
// gettext_config:{"id":"add-timeout-$1-$2"}
T : [ 'Add timeout %1: [%2]', time_message(timeout),
URL_to_fetch ]
}, 2, 'get_URL_node');
} else if (timeout) {
library_namespace.warn([ 'get_URL_node: ', {
// gettext_config:{"id":"invalid-timeout-$1"}
T : [ 'Invalid timeout: %1', timeout ]
} ]);
}
library_namespace.debug('set onerror: '
+ (options.onfail ? 'user defined' : 'default handler'), 3,
'get_URL_node');
request.on('error', _onfail);
// 遇到 "Unhandled 'error' event",或許是 print 到 stdout 時出錯了,不一定是本函數的問題。
// debug error: socket parse error
// CloudFlare 遇到 HPE_INVALID_CONSTANT可能是因為需要 encodeURI(url)。
if (false && library_namespace.is_debug(6)) {
request.on('socket', function(socket) {
if (socket.parser) {
socket.parser._execute = socket.parser.execute;
socket.parser.execute = function(d) {
console.log(d.toString());
socket.parser._execute(d);
};
}
// console.log('-----------------------------------');
// console.log(socket.parser.execute);
});
}
var start_time = (new Date).getTime();
request.end();
}
/**
* default user agent. for some server, (e.g., tools.wmflabs.org)
* <q>Requests must have a user agent</q>.
*
* @see https://meta.wikimedia.org/wiki/User-Agent_policy
*
* @type {String}
*/
get_URL_node.default_user_agent = library_namespace.Class + '/'
+ library_namespace.version + ' (https://github.com/kanasimi/CeJS)';
// 逾時ms數: 20 minutes
get_URL_node.default_timeout = 20 * 60 * 1000;
get_URL_node.connects_limit = 100;
get_URL_node.get_status = function(item) {
var status = {
connections : get_URL_node_connections,
requests : get_URL_node_requests
};
return item ? status[item] : status;
};
// setup/reset node agent.
function setup_node(type, options) {
if (!is_nodejs)
return;
if (_.get_URL !== get_URL_node) {
// 初始化。
node_url = require('url');
node_http = require('http');
node_https = require('https');
node_zlib = require('zlib');
try {
node_http2 = require('http2');
} catch (e) {
// Is old version. Added in: v8.4.0
}
_.get_URL = library_namespace
.copy_properties(get_URL, get_URL_node);
}
if (type !== undefined) {
if (typeof type === 'string')
type = /^https/i.test(type);
var agent = type ? new node_https.Agent(options)
: new node_http.Agent(options);
if (options && options.as_default) {
if (type) {
node_https_agent = agent;
} else {
node_http_agent = agent;
}
}
return agent;
}
node_http_agent = new node_http.Agent;
node_https_agent = new node_https.Agent;
// 不需要。
// node_http_agent.maxSockets = 1;
// node_https_agent.maxSockets = 1;
}
_.setup_node_net = setup_node;
// CeL.application.net.Ajax.setup_node_net();
// library_namespace.application.net.Ajax.setup_node_net();
setup_node();
// ---------------------------------------------------------------------//
function parse_proxy_server(proxy_server) {
// console.log(proxy_server);
proxy_server = library_namespace.URI(proxy_server);
proxy_server.proxy = proxy_server.href;
// console.log(proxy_server);
return proxy_server;
if (typeof proxy_server !== 'string') {
return proxy_server;
}
// href=protocol:(//)?username:password@hostname:port/path/filename?search#hash
// 代理伺服器 proxy_server: "username:password@hostname:port"
// [ all, protocol, username, password, hostname, port ]
var matched = proxy_server
.match(/^(?:(https?:)\/\/)?(?:([^:@]+)(?::([^@]*))?@)?([^:@]+)(?::(\d{1,5}))?$/);
if (!matched) {
return false;
}
matched = {
proxy : matched[0],
protocol : matched[1],
username : matched[2],
password : matched[3],
hostname : matched[4],
port : +matched[5]
};
return matched;
}
function get_proxy_URL(proxy_server, URL_options_to_fetch, URL_to_fetch) {
var proxy_URL = {
host : proxy_server.hostname,
port : proxy_server.port
|| library_namespace.net.port_of_protocol[URL_options_to_fetch.protocol
.replace(/:$/, '')]
|| library_namespace.net.port_of_protocol.https,
path : URL_to_fetch,
protocol : URL_options_to_fetch.protocol,
// method: 'GET',
headers : {
Host : URL_options_to_fetch.host
}
};
if (proxy_server.agent) {
proxy_URL.agent = proxy_server.agent;
}
if (proxy_server.username) {
// https://developer.mozilla.org/zh-TW/docs/Web/HTTP/Authentication
// https://developer.mozilla.org/zh-TW/docs/Web/HTTP/Headers/Proxy-Authorization
proxy_URL.headers['Proxy-Authorization'] = 'Basic ' + Buffer.from(
// proxy.auth
proxy_server.username + ':' + (proxy_server.password || '')
//
).toString('base64');
}
return proxy_URL;
}
// http://luoxia.me/code/2017/07/16/%E8%81%8A%E8%81%8AAgent&Proxy/
// https://github.com/TooTallNate/node-https-proxy-agent
/**
* get https:// through proxy 用於取得https網站。
*
* @see https://www.vanamco.com/2014/06/24/proxy-requests-in-node-js/
* https://gist.github.com/matthias-christen/6beb3b4dda26bd6a221d
*
* modify from:
*
* HTTPS Agent for node.js HTTPS requests via a proxy.
* blog.vanamco.com/connecting-via-proxy-node-js/
*/
function HttpsProxyAgent(proxy_server, options) {
if (!(this instanceof HttpsProxyAgent))
return new HttpsProxyAgent(proxy_server, options);
node_https.Agent.call(this, options || {});
this.options = Object.assign({}, options);
// href=protocol:(//)?username:password@hostname:port/path/filename?search#hash
// 代理伺服器 proxy_server: "username:password@hostname:port"
proxy_server = library_namespace.URI(proxy_server);
if (!proxy_server) {
// gettext_config:{"id":"must-specify-proxy-server-hostname-port"}
throw new Error('Must specify proxy server: hostname:port')
}
this.proxy_server = proxy_server;
// https://github.com/nodejs/node/blob/master/lib/net.js
// function connect(...args) { }
this.createConnection = function connect_proxy_server(options, callback) {
// do a CONNECT request
var request = Object.assign(get_proxy_URL(proxy_server, options,
options.host + ':' + options.port), {
method : 'CONNECT'
});
// console.log(request);
// 此時若有 agent會是 https。但是待會要用 http connect。
delete request.agent;
delete request.protocol;
if (proxy_server.agent)
set_cookie_to_URL_object(request, proxy_server.agent);
library_namespace.debug('Connect to ' + request.path, 2,
'HttpsProxyAgent.createConnection');
request = node_http.request(request);
request.on('connect', function(response, socket, headers) {
var tls = require('tls');
// https://github.com/nodejs/node/issues/27384
// node.js v12 disable TLS v1.0 and v1.1 by default
// tls.DEFAULT_MIN_VERSION = 'TLSv1';
// a tls.TLSSocket object
var tls_socket = tls.connect({
host : options.host,
socket : socket
}, function() {
callback(null, tls_socket);
});
});
request.on('error', callback);
request.end();
}
}
/**
* <code>
node_https.request({
// like you'd do it usually...
host : 'twitter.com',
port : 443,
method : 'GET',
path : '/',
// set proxy
agent : new CeL.HttpsProxyAgent('localhost:8080')
}, function(resonse) {
resonse.on('data', function(data) {
console.log(data.toString());
});
}).end();
</code>
*/
_.HttpsProxyAgent = HttpsProxyAgent;
var node_util;
if (is_nodejs) {
node_util = require('util');
node_util.inherits(HttpsProxyAgent, node_https.Agent);
}
// https://github.com/nodejs/node/blob/master/lib/_http_agent.js
HttpsProxyAgent.prototype.getName = function getName(options) {
var name = (options.host || 'localhost') + ':'
//
+ (options.port || '') + ':' + (options.path || '');
return name;
};
// Almost verbatim copy of http.Agent.addRequest
// https://github.com/nodejs/node/blob/master/lib/_http_agent.js
// Agent.prototype.addRequest = function addRequest(req, options
HttpsProxyAgent.prototype.addRequest = function addRequest(request, options) {
// Get the key for a given set of request options
// Agent.prototype.getName
var name = this.getName(options);
if (!this.sockets[name])
this.sockets[name] = [];
// Do not use this.freeSockets
if (this.sockets[name].length < this.maxSockets) {
options.request = request;
// If we are under maxSockets create a new one.
this.createSocket(name, options, function(error, tls_socket) {
if (error) {
process.nextTick(function emitErrorNT(emitter, error) {
emitter.emit('error', error);
}, request, error);
return;
}
// setRequestSocket(agent, request, socket);
request.onSocket(tls_socket);
// TODO: set this_agent.options.timeout
});
} else {
// We are over limit so we'll add it to the queue.
if (!this.requests[name])
this.requests[name] = [];
this.requests[name].push(request);
// lost options???
}
};
// Almost verbatim copy of http.Agent.createSocket
// https://github.com/nodejs/node/blob/master/lib/_http_agent.js
// Agent.prototype.createSocket
HttpsProxyAgent.prototype.createSocket = function createSocket(name,
options, callback) {
var this_agent = this;
options = Object.assign({}, options, this.options);
options.servername = options.host;
if (options.request) {
var hostHeader = options.request.getHeader('host');
if (hostHeader)
options.servername = hostHeader.replace(/:.*$/, '');
}
var called = false;
function oncreate(error, tls_socket) {
if (called)
return;
called = true;
if (error) {
error.message += ' while connecting to HTTP(S) proxy server '
+ this_agent.hostname + ':' + this_agent.port;
if (options.request)
options.request.emit('error', error);
else
throw error;
return;
}
var name = this_agent.getName(options);
if (!this_agent.sockets[name])
this_agent.sockets[name] = [];
this_agent.sockets[name].push(tls_socket);
// ------------------------
// function installListeners(agent, socket, options)
var onFree = function onFree() {
this_agent.emit('free', tls_socket, options);
};
var onClose = function onClose(error) {
/**
* This is the only place where sockets get removed from the
* Agent.
*
* If you want to remove a socket from the pool, just close it.
*
* All socket errors end in a close event anyway.
*/
this_agent.removeSocket(tls_socket, options);
};
var onRemove = function onRemove() {
/**
* We need this function for cases like HTTP 'upgrade' (defined
* by WebSockets) where we need to remove a socket from the pool
* because it'll be locked up indefinitely
*/
this_agent.removeSocket(tls_socket, options);
tls_socket.removeListener('close', onClose);
tls_socket.removeListener('free', onFree);
tls_socket.removeListener('agentRemove', onRemove);
};
tls_socket.on('free', onFree);
tls_socket.on('close', onClose);
tls_socket.on('agentRemove', onRemove);
// ------------------------
// assert: error === null
callback(error, tls_socket);
}
// call connect_proxy_server()
this_agent.createConnection(options, oncreate);
};
// ---------------------------------------------------------------------//
// https://github.com/TooTallNate/node-socks-proxy-agent
function SocksProxyAgent() {
TODO;
}
// ---------------------------------------------------------------------//
// TODO: for non-nodejs
/** {Object|Function}fs in node.js */
var node_fs;
try {
if (is_nodejs) {
// @see https://nodejs.org/api/fs.html
node_fs = require('fs');
}
if (typeof node_fs.readFile !== 'function') {
throw true;
}
} catch (e) {
library_namespace.debug([ this.id + ': ', {
// gettext_config:{"id":"there-is-no-`fs`-package-for-node.js-so-there-is-no-file-operation-function-for-node"}
T : '無 node.js 之 `fs` 套件,因此不具備 node 之檔案操作功能。'
} ]);
if (false) {
// enumerate for get_URL_cache_node
// 模擬 node.js 之 fs以達成最起碼的效果即無 cache 功能的情況)。
node_fs = {
readFile : function(file_name, charset, callback) {
callback(true);
},
writeFile : library_namespace.null_function
};
}
}
/**
* <code>
cache 相關函數:
@see
application.storage.file.get_cache_file
application.OS.Windows.file.cacher
application.net.Ajax.get_URL_cache
application.net.wiki wiki_API.cache() CeL.wiki.cache()
</code>
*/
/**
* cache 作業操作之輔助套裝函數。
*
* 注意: 若執行 onload() 時沒提供 XMLHttp表示採用 cache。
*
* TODO: 以 HEAD 檢測,若有新版本則不採用 cache。
*
* @param {String|Object}URL
* 欲請求之目的 URL or options
* @param {Function}[onload]
* callback when successful loaded. onload(data, error, XMLHttp)
* @param {Object}[options]
* 附加參數/設定特殊功能與選項
*/
function get_URL_cache_node(URL, onload, options) {
if (typeof options === 'string') {
// auto-detecting
options = /\.[a-z\d\-]+$/i.test(options) ? {
file_name : options
} : /[\\\/]+$/i.test(options)
// || 也可以測試是不是目錄、此目錄是否存在。
? {
directory : options
} : {
encoding : options
};
} else if (!library_namespace.is_Object(options)) {
// 前置處理。
options = Object.create(null);
}
var file_name = options.file_name,
/** {String}file encoding for fs of node.js. */
encoding = 'encoding' in options ? options.encoding
: get_URL_cache_node.encoding;
if (!file_name && (file_name = decodeURI(URL).match(/[^\/]+$/))) {
file_name = file_name[0];
if (library_namespace.HTML_to_Unicode) {
// 去掉 "&amp;" 之類。
file_name = library_namespace.HTML_to_Unicode(file_name);
}
file_name = library_namespace.to_file_name(file_name);
library_namespace.debug([ {
// gettext_config:{"id":"get-filename-from-url-$1"}
T : [ '自 URL 取得檔名:%1', URL ]
}, '\n→ ' + file_name ], 1, 'get_URL_cache_node');
}
if (typeof options.file_name_processor === 'function') {
file_name = options.file_name_processor(file_name);
}
if (!file_name) {
// gettext_config:{"id":"no-file-name-specified"}
onload(undefined, new Error('No file name specified.'));
return;
}
if (options.directory) {
file_name = library_namespace.append_path_separator(
options.directory, file_name);
}
var file_status;
try {
file_status = node_fs.statSync(file_name);
} catch (e) {
// TODO: handle exception
}
if (!options.get_contents && options.web_resource_date && file_status) {
// download newer only
if ((file_status.mtimeMs || file_status.mtime)
//
- Date.parse(options.web_resource_date) > -1) {
library_namespace.debug('File on web ('
+ options.web_resource_date
+ ') is not newer than local file ('
+ file_status.mtime + '): ' + file_name + '', 1,
'get_URL_cache_node');
onload(undefined, _.get_URL_cache.NO_NEWS);
return;
}
}
library_namespace.debug([
{
// gettext_config:{"id":"download-$1"}
T : [ '下載 %1', URL ]
},
'\n→ ',
{
// gettext_config:{"id":"$1-(file-encoding-$2-charset-$3)"}
T : [ '%1 (file encoding %2, charset %3)', file_name,
encoding, options.charset ]
} ], 1, 'get_URL_cache_node');
node_fs.readFile(file_name, encoding,
//
function(error, data) {
// console.trace([ error, !!data, options.reget ]);
// 警告: 對於從 HTTP 標頭獲得文件名的情況,就算不設定 options.reget 也沒用,還是會重新獲取檔案。
// options.force_download
if (!options.reget) {
if (!error && options.web_resource_date && file_status) {
// download newer only
if ((file_status.mtimeMs || file_status.mtime)
//
- Date.parse(options.web_resource_date) > -1) {
// No new file on web.
onload(data);
return;
}
}
if (!error && !options.preserve_newer
// 若是容許空內容,應該特別指定 options.allow_empty。
&& (data || options.allow_empty)) {
library_namespace.debug({
// gettext_config:{"id":"using-cached-data"}
T : 'Using cached data.'
}, 3, 'get_URL_cache_node');
library_namespace.debug('Cached data: ['
+ data.slice(0, 200) + ']...', 5,
'get_URL_cache_node');
// TODO: use cached_status
onload(data);
return;
}
library_namespace.debug({
// gettext_config:{"id":"no-valid-cached-data.-try-to-get-data-(again)"}
T : 'No valid cached data. Try to get data (again)...'
}, 3, 'get_URL_cache_node');
}
_.get_URL(URL, function(XMLHttp, error) {
if (error) {
library_namespace.error([ 'get_URL_cache_node.cache: ', {
// gettext_config:{"id":"got-error-when-retrieving-$1-$2"}
T : [ 'Got error when retrieving [%1]: %2',
//
URL, localize_error(error) ]
} ]);
// WARNING: XMLHttp 僅在重新取得 URL 時提供。
onload(undefined, error, XMLHttp);
return;
}
// .buffer: node only.
data = !encoding && XMLHttp.buffer || XMLHttp.response
|| XMLHttp.responseText;
// 資料事後處理程序 (post-processor):
// 將以 .postprocessor() 的回傳作為要處理的資料。
if (typeof options.postprocessor === 'function') {
data = options.postprocessor(data, XMLHttp);
}
if (options.file_name_processor) {
file_name = options.file_name_processor(file_name,
// header_filename
XMLHttp.filename);
} else if (XMLHttp.filename) {
if (false) {
console.log([ options.directory, options.file_name,
XMLHttp.filename ]);
console.log(XMLHttp.headers);
}
if (!options.file_name) {
file_name = (options.directory || '')
// 若是沒有特別設置檔名則改採用header裡面的檔名。
+ XMLHttp.filename;
library_namespace.info([
'get_URL_cache_node: ',
{
// gettext_config:{"id":"got-file-name-from-http-header-$1"}
T : [ 'Got file name from HTTP header: %1',
XMLHttp.filename ]
} ]);
} else if (!options.file_name.endsWith(XMLHttp.filename)) {
library_namespace.info([ 'get_URL_cache_node: ', {
T : [
// gettext_config:{"id":"set-file-name-$1-file-name-from-header-$2"}
'Set file name: [%1], file name from header: [%2].'
//
, options.file_name, XMLHttp.filename ]
} ]);
}
}
var URL_date = XMLHttp.headers['date'];
if (URL_date && options.preserve_newer && file_status) {
// data.length === stat.size
if ((file_status.mtimeMs || file_status.mtime)
//
- Date.parse(URL_date) > -1) {
// Local file is newer than file on web.
onload(data.toString(), undefined, XMLHttp);
return;
}
}
/**
* 寫入cache。
*
* 若出現錯誤則不寫入cache。
*/
if (data && /[^\\\/]$/.test(file_name)) {
XMLHttp.cached_file_path = file_name;
if (!file_status) {
try {
file_status = node_fs.statSync(file_name);
if (file_status && !options.reget) {
library_namespace.info([
'get_URL_cache_node.cache: ',
{
// gettext_config:{"id":"find-that-the-file-exists-after-get-the-file-from-web-$1-.-do-not-overwrite-it"}
T : [ '重新獲取檔案後發現原檔案已存在,跳過不覆寫:[%1]',
file_name ]
} ]);
onload(data.toString(), undefined, XMLHttp);
return;
}
} catch (e) {
// TODO: handle exception
}
}
if (!options.no_write_info) {
library_namespace.info([
'get_URL_cache_node.cache: ',
{
// gettext_config:{"id":"write-data-to-cache-file-$1"}
T : [ 'Write data to cache file [%1].',
file_name ]
} ]);
library_namespace.debug({
// gettext_config:{"id":"the-data-to-cache-$1"}
T : [ 'The data to cache: %1...',
//
data && JSON.stringify(data).slice(0, 190) ]
}, 3, 'get_URL_cache_node');
}
try {
node_fs.writeFileSync(file_name, data, encoding);
} catch (error) {
onload(data.toString(), error, XMLHttp);
return;
}
}
// set file modify date
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date
// https://tools.ietf.org/html/rfc7231#section-7.1.1.2
if (URL_date) {
try {
// The "Date" header field represents the date
// and time at which the message was originated
// fs.utimesSync(path, atime, mtime)
// atime: the last time this file was accessed
node_fs.utimesSync(file_name, new Date,
// mtime: the last time this file was modified
URL_date);
} catch (e) {
// TODO: handle exception
}
}
// Warning: 已經有些程式碼預設會回傳 {String}
onload(data.toString(), undefined, XMLHttp);
},
// character encoding of HTML web page
// is different from the file we want to save to
options.charset, options.post_data, options.get_URL_options);
});
}
/** {String}預設 file encoding for fs of node.js。 */
get_URL_cache_node.encoding = 'utf8';
if (is_nodejs) {
_.get_URL_cache = get_URL_cache_node;
_.get_URL_cache.NO_NEWS = typeof Symbol === 'function' ? Symbol('no news')
: {
'no news' : true
};
}
// ---------------------------------------------------------------------//
/**
* defective polyfill for W3C fetch API
*
* @since 2018/10/16 17:47:12
* @deprecated
*/
function node_fetch(input, init) {
// TODO: input is a Request object.
var url = input instanceof URL ? input : new URL(input.toString()), options = library_namespace
.new_options(init);
function executor(resolve, reject) {
function callback(response) {
if ((response.statusCode / 100 | 0) === 3
&& response.headers.location
&& response.headers.location !== url.toString()) {
try {
// request.abort();
request.destroy();
} catch (e) {
}
Object.assign(options, {
redirected : true,
initial_URL : options.initial_URL || input
});
url = new URL(response.headers.location, url);
library_namespace.debug({
// gettext_config:{"id":"$1-redirecting-to-$2-←-$3"}
T : [ '%1 Redirecting to [%2] ← [%3]',
response.statusCode, url.toString(), input ]
}, 1, 'fetch');
node_fetch(url, options);
return;
}
/** {Array} [ {Buffer}, {Buffer}, ... ] */
var data = [], length = 0;
response.on('data', function(chunk) {
// {Buffer}chunk
length += chunk.length;
library_namespace.debug('receive BODY '
//
+ chunk.length + '/' + length + ': ' + url, 4, 'fetch');
data.push(chunk);
});
response.on('end', function() {
library_namespace.debug('end(): ' + url, 2, 'fetch');
// console.log('No more data in response: ' + url);
// it is faster to provide the length explicitly.
data = Buffer.concat(data, length);
var result_Object = {
// https://developer.mozilla.org/zh-TW/docs/Web/API/Response
// https://nodejs.org/api/http.html#http_http_get_options_callback
url : options.initial_URL || input,
headers : response.headers,
status : response.statusCode,
statusText : response.statusMessage,
ok : (response.statusCode / 100 | 0) === 2,
redirected : !!options.redirected,
useFinalURL : url.toString(),
_buffer : data,
// TODO: body : new ReadableStream()
// methods of
// https://developer.mozilla.org/en-US/docs/Web/API/Body
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream
text : function text() {
try {
return Promise.resolve(
//
this._buffer.toString());
} catch (e) {
return Promise.reject(e);
}
},
json : function json() {
return this.text().then(JSON.parse);
},
arrayBuffer : function arrayBuffer() {
return Promise.resolve(this._buffer.buffer);
}
};
resolve(result_Object);
});
}
// TODO: add normal headers
// CloudFlare 必須設定好headers才能才會才允許回傳資料。
if (library_namespace.is_debug(9)
&& library_namespace.env.has_console) {
console.trace([ url.toString(), options ]);
}
// https://nodejs.org/api/http.html
var request = url.protocol === 'http:' ? node_http.request(url
.toString(), options, callback) : node_https.request(url
.toString(), options, callback);
request.on('error', reject);
if (options.body)
request.write(options.body);
request.end();
}
return new Promise(executor);
}
/**
* defective polyfill for W3C fetch API
*
* 必須額外設定 credentials。
*
* TODO: fetch 預設上不傳送或接收任何 cookies如果網站依賴 session 會導致請求回傳未經認證,需要使用 cookies
*
* @examples <code>
var fetch = CeL.fetch;
fetch(url).then(function(response) {
return response.json();
}).then(function(json) {
console.log(json);
});
fetch(url).then(function(response) {
return response.text();
}).then(function(html) {
console.log(html);
});
</code>
*
* @see 20181016.import_earthquake_shakemap.js
* @see https://fetch.spec.whatwg.org/#fetch-method
* https://developer.mozilla.org/zh-TW/docs/Web/API/Fetch_API
* https://github.com/node-fetch/node-fetch
*
* @since 2021/8/4 6:6:45
*/
function fetch__get_URL(input, init) {
function executor(resolve, reject) {
function callback(XMLHttp, error) {
if (error) {
reject(error);
return;
}
// console.trace(XMLHttp);
var result_Object = {
// https://developer.mozilla.org/zh-TW/docs/Web/API/Response
// https://nodejs.org/api/http.html#http_http_get_options_callback
headers : XMLHttp.headers,
ok : (XMLHttp.status / 100 | 0) === 2,
redirected : XMLHttp.redirected,
status : XMLHttp.status,
statusText : XMLHttp.statusText,
// 重定向後獲得的最終 URL。
url : XMLHttp.responseURL,
useFinalURL : true,
_buffer : XMLHttp.buffer,
// TODO: body : new ReadableStream()
// methods of
// https://developer.mozilla.org/en-US/docs/Web/API/Body
// https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream
text : function text() {
try {
return Promise.resolve(
//
this._buffer.toString());
} catch (e) {
return Promise.reject(e);
}
},
json : function json() {
return this.text().then(JSON.parse);
},
arrayBuffer : function arrayBuffer() {
return Promise.resolve(this._buffer.buffer);
}
};
resolve(result_Object);
}
// CloudFlare 必須設定好 headers 才能才會才允許回傳資料。
// get_URL() 可自動設定 headers。
_.get_URL(input, callback, null, init && init.body, Object.assign({
// headers : { 'User-Agent' : '' },
onfail : reject,
agent : true
}, init));
}
return new Promise(executor);
}
if (is_nodejs) {
_.fetch = fetch__get_URL;
}
// ---------------------------------------------------------------------//
// export 導出.
return (_// JSDT:_module_
);
}