/** * @name CeL function for Ajax (Asynchronous JavaScript and XML) * @fileoverview 本檔案包含了模擬WWW客戶端發送HTTP/HTTPS請求用的 functions。 * @since 2015/1/1 */ 'use strict'; // 'use asm'; // -------------------------------------------------------------------------------------------- // 不採用 if 陳述式,可以避免 Eclipse JSDoc 與 format 多縮排一層。 typeof CeL === 'function' && CeL.run({ // module name name : 'application.net.Ajax', // Promise for fetch() require : 'data.code.compatibility.' // library_namespace.copy_properties() + '|data.native' // MIME_of() + '|application.net.MIME.' // for CeL.to_file_name(), CeL.URI, CeL.Search_parameters + '|application.net.', // 設定不匯出的子函式。 // no_extend : '*', // 為了方便格式化程式碼,因此將 module 函式主體另外抽出。 code : module_code }); function module_code(library_namespace) { function time_message(millisecond) { if (library_namespace.age_of) { return library_namespace.age_of(0, millisecond, { digits : 1 }); } return millisecond % 1000 === 0 ? millisecond / 1000 + 's' : millisecond + 'ms'; } // "Error: socket hang up" {code: 'ECONNRESET'} // "Error: connect ETIMEDOUT 1.1.1.1:80" // {errno:'ETIMEDOUT', code: 'ETIMEDOUT', address: '125.89.70.31', port:80 } // Error: connect ECONNREFUSED 127.0.0.1:443 // "Error: read ECONNRESET" // {errno: 'ECONNRESET', code: 'ECONNRESET', syscall: 'read'} // Error: getaddrinfo ENOTFOUND domain // ERROR_BAD_STSTUS // Error: Timeout 30s function localize_error(error) { var message = String(error); if (library_namespace.gettext) { // 處理特別的錯誤訊息。 var matched = message .match(/^(Error: (?:(?:connect|getaddrinfo) E[A-Z]+|Timeout) )(.+)$/); if (matched) { message = [ matched[1] + '%1', matched[2] ]; } message = Array.isArray(message) // gettext_config:{"id":"error-connect-etimedout-$1","mark_type":"combination_message_id"} // gettext_config:{"id":"error-connect-econnrefused-$1","mark_type":"combination_message_id"} // gettext_config:{"id":"error-getaddrinfo-enotfound-$1","mark_type":"combination_message_id"} // gettext_config:{"id":"error-timeout-$1","mark_type":"combination_message_id"} ? library_namespace.gettext.apply(null, message) // gettext_config:{"id":"error-socket-hang-up","mark_type":"combination_message_id"} // gettext_config:{"id":"error-read-econnreset","mark_type":"combination_message_id"} // gettext_config:{"id":"error-write-econnaborted","mark_type":"combination_message_id"} // gettext_config:{"id":"error-unexpected-end-of-file","mark_type":"combination_message_id"} : library_namespace.gettext(message); } return message; } var /** {Number}未發現之index。 const: 基本上與程式碼設計合一,僅表示名義,不可更改。(=== -1) */ NOT_FOUND = ''.indexOf('_'); /** * null module constructor * * @class web Ajax 的 functions */ var _// JSDT:_module_ = function() { // null module constructor }; /** * for JSDT: 有 prototype 才會將之當作 Class */ _// JSDT:_module_ .prototype = {}; // ---------------------------------------------------------------------// // XMLHttp set ajax通信処理ライブラリ ================== /** * to use: include in front: way1(good: 以reg代替functionPath!): // [function.js]_iF // [function.js]End way2(old): // [function.js]getU,functionPath,'eval(getU(functionPath));' // [function.js]End old: function getU(p){var o;try{o=new ActiveXObject('Microsoft.XMLHTTP');}catch(e){o=new XMLHttpRequest();}if(o)with(o){open('GET',p,false),send(null);return responseText;}} */ /** * JScript or .wsh only, 能 encode. * * @param {String}page_url * page url * @param {String}[charset] * character encoding of HTML web page. e.g., 'UTF-8', big5, * euc-jp, ... * @param POST_text * POST text * * @returns {String} * @see http://neural.cs.nthu.edu.tw/jang/books/asp/getWebPage.asp?title=10-1%20%E6%8A%93%E5%8F%96%E7%B6%B2%E9%A0%81%E8%B3%87%E6%96%99 */ function get_page(page_url, charset, POST_text) { try { // may cause error var X = new ActiveXObject('Microsoft.XMLHTTP'), AS; X.open(POST_text ? 'POST' : 'GET', page_url, false); // POST need this X.setRequestHeader("Content-Type", "application/x-www-form-urlencoded"); // Download the file X.send(POST_text || null); AS = new ActiveXObject("ADODB.Stream"); // 可同時進行讀寫 AS.Mode = 3; // 以二進位方式操作 AS.Type = 1; // 開啟物件 AS.Open(); // 將 binary 的資料寫入物件內 may error AS.Write(X.responseBody); AS.Position = 0; // 以文字模式操作 AS.Type = 2; // 設定編碼方式 if (charset) AS.Charset = charset; // 將物件內的文字讀出 X = AS.ReadText(); // Release memory. 釋放被占用的記憶體. AS = null; return X; } catch (e) { library_namespace.warn('get_page: ' + e.message); } } // ---------------------------------------------------------------------// var KEY_URL = typeof Symbol === 'function' ? Symbol('URL') : '\0URL to fetch'; /** * * @param URL_to_fetch * @param search * @param hash * @returns * * @inner */ function set_parameters_and_hash(URL_to_fetch, search, hash) { // URL_to_fetch = library_namespace.URI(URL_to_fetch); // assert: library_namespace.is_URI(URL_to_fetch) if (hash || hash === '') { if (Object.defineProperty[KEY_not_native] && !/^#/.test(hash)) hash = '#' + hash; URL_to_fetch.hash = hash; } URL_to_fetch.search_params.set_parameters(search); // console.trace(URL_to_fetch.toString(charset)); return URL_to_fetch; } function normalize_URL_to_fetch(URL_to_fetch, charset, options) { // console.trace(URL_to_fetch); // https://developer.mozilla.org/en-US/docs/Web/API/URL // [ origin + pathname, search, hash ] // hrer = [].join('') if (Array.isArray(URL_to_fetch)) { URL_to_fetch = set_parameters_and_hash(library_namespace .URI(URL_to_fetch[0]), URL_to_fetch[1], URL_to_fetch[2]); if (charset) URL_to_fetch.charset = charset; } else { // 當輸入 {URL} 時,node_https.request() 會將 {URL} 轉成 // {Object}options,不會考慮額外選項 (headers, ...)。 // 且必須處理 charset,乾脆直接將 {URL} 轉成尋常 plain object / {URI}。 // https://nodejs.org/api/http.html#http_http_request_url_options_callback // If url is a string, it is automatically parsed with new URL(). If // it is a URL object, it will be automatically converted to an // ordinary options object. // console.log(URL_to_fetch); URL_to_fetch = library_namespace.URI(URL_to_fetch, null, { charset : charset }); // console.trace(URL_to_fetch); } // assert: library_namespace.is_URI(URL_to_fetch) // console.trace(URL_to_fetch); if (options.search || options.hash) { URL_to_fetch = set_parameters_and_hash(URL_to_fetch, options.search, options.hash); } library_namespace.debug({ T : [ 'Fetching URL: %1', '{' + (typeof URL_to_fetch) + '} [' // + (typeof URL_to_fetch === 'string' ? URL_to_fetch // : URL_to_fetch && URL_to_fetch[KEY_URL] // || URL_to_fetch.toString(charset)) + ']' ] }, 1, 'normalize_URL_to_fetch'); return URL_to_fetch; } if (false) // default arguments var get_URL_arguments = { URL : '', charset : '', // HTTP方法,如"GET", "POST", HEAD, "PUT", "DELETE"等。 method : 'GET', post_data : {}, async : true, // user name. 驗證用使用者名稱。 user : '', // 驗證用密碼。 password : '', // header headers : { contentType : 'text/xml' }, // location.search search : { contentType : 'text/xml' }, // location.hash hash : '', mime : 'text/xml', // onreadystatechange onchange : function() { }, timeout : 0, onfail : function(error) { this.status; }, onload : function() { } }; // XMLHttp.readyState 所有可能的值如下: // 0 還沒開始 // 1 讀取中 Sending Data // 2 已讀取 Data Sent // 3 資訊交換中 interactive: getting data // 4 一切完成 Completed var readyState_done = 4, // document_head = library_namespace.is_WWW(true) && (document.head || document.getElementsByTagName('head')[0]); /** * 讀取 URL via XMLHttpRequest。 * * @param {String|Object}URL_to_fetch * 欲請求之目的 URL or options * @param {Function}[onload] * callback when successful loaded * @param {String}[charset] * character encoding of HTML web page. e.g., 'UTF-8', big5, * euc-jp, ... * @param {String|Object}[post_data] * text data to send when method is POST * @param {Object}[options] * 附加參數/設定選擇性/特殊功能與選項 * * TODO: 代理伺服器 using proxy server * * @see https://developer.mozilla.org/zh-TW/docs/DOM/XMLHttpRequest * http://msdn.microsoft.com/en-us/library/ie/ms535874.aspx */ function get_URL(URL_to_fetch, onload, charset, post_data, options) { // 前導作業。 if (library_namespace.is_Object(charset)) { post_data = charset; charset = null; } // 正規化並提供可隨意改變的同內容參數,以避免修改或覆蓋附加參數。 options = library_namespace.new_options(options); // ------------------------------------------------------ if (library_namespace.is_Object(URL_to_fetch) && URL_to_fetch[KEY_URL]) { Object.assign(options, URL_to_fetch); // 注意: options.onload 另有用途! // https://xhr.spec.whatwg.org/#handler-xhr-onloadstart // onload = options.onload || onload; post_data = options.post || post_data; charset = options.charset || charset; URL_to_fetch = options[KEY_URL]; } URL_to_fetch = normalize_URL_to_fetch(URL_to_fetch, charset, options); // assert: library_namespace.is_URI(URL_to_fetch) if (typeof onload === 'object') { library_namespace.debug( 'Trying to JSONP, insert page, need callback.', 3, 'get_URL'); // library_namespace.run(URL_to_fetch); for ( var callback_param in onload) { library_namespace.debug('Trying (' + (typeof onload[callback_param]) + ') [' + callback_param + '] = [' + onload[callback_param] + ']', 3, 'get_URL'); if (callback_param && typeof onload[callback_param] === 'function') { var callback_name, node = document.createElement('script'); for (charset = 0; (callback_name = 'cb' + charset) in library_namespace;) charset++; library_namespace[callback_name] = function(data) { library_namespace.debug('[' + URL_to_fetch + ']: callback 完自動移除 .js。', 2, 'get_URL'); document_head.removeChild(node); // Release memory. 釋放被占用的記憶體. node = null; delete library_namespace[callback_name]; onload[callback_param](data); }; // callback_param: callback parameter URL_to_fetch.search_params[callback_param] = library_namespace.Class + '.' + callback_name; node.src = URL_to_fetch.toString(); library_namespace.debug('Use script node: [' + node.src + ']', 3, 'get_URL'); document_head.appendChild(node); return; } } library_namespace.debug('Skip JSONP. No callback specified.', 3, 'get_URL'); } if (post_data && !options.form_data) { post_data = library_namespace.Search_parameters(post_data) .toString(charset); } if (!onload && typeof options.onchange === 'function') { onload = function() { options.onchange(readyState_done, XMLHttp); }; } if (options.async === false && onload || typeof onload !== 'function') { onload = false; } /** * The XMLHttpRequest object can't be cached. */ var XMLHttp = library_namespace.new_XMLHttp(); try { // IE:404 會 throw error, timeout 除了 throw error, // 還會 readystatechange; // Gecko 亦會 throw error // IE 10 中,local file 光 .open() 就 throw 了。 XMLHttp.open(options.method || (post_data ? 'POST' : 'GET'), URL_to_fetch.toString(), !!onload, options.user || '', options.password || ''); // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/response // XMLHttp.responseType = 'blob'; if (options.timeout > 0 && !onload) { // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/timeout XMLHttp.timeout = options.timeout; if (typeof options.onfail === 'function') XMLHttp.ontimeout = function(e) { options.onfail.call(XMLHttp, e || 'Timeout'); }; } // TODO: 處理有 onload 下之 timeout 逾時ms數 // Ajax 程式應該考慮到 server 沒有回應時之處置 if (library_namespace.is_Object(options.headers) // https://developer.mozilla.org/zh-TW/docs/Web/API/XMLHttpRequest/setRequestHeader // `!!XMLHttp.setRequestHeader` will throw @ HTA (HTML Application) && ('setRequestHeader' in XMLHttp)) { Object.keys(options.headers).forEach(function(key) { XMLHttp.setRequestHeader(key, options.headers[key]); }); } if (options.mime) { // ignore charset! charset = options.mime; } else if (charset) { // old: 'text/xml;charset=' + charset // 但這樣會被當作 XML 解析,產生語法錯誤。 // TODO: try: // 'text/'+(/\.x(ht)?ml$/i.test(URL_to_fetch)?'xml':'plain')+';charset=' // + charset; charset = 'application/json;charset=' + charset; } // 有些版本的 Mozilla 瀏覽器在伺服器送回的資料未含 XML mime-type // 檔頭(header)時會出錯。為了避免這個問題,可以用下列方法覆寫伺服器傳回的檔頭,以免傳回的不是 text/xml。 // http://squio.nl/blog/2006/06/27/xmlhttprequest-and-character-encoding/ // http://www.w3.org/TR/XMLHttpRequest/ search encoding if (charset && XMLHttp.overrideMimeType) XMLHttp.overrideMimeType(charset); if (onload) { XMLHttp.onreadystatechange = function() { if (XMLHttp.readyState === readyState_done) return onload(XMLHttp); if (0 < XMLHttp.readyState && XMLHttp.readyState < readyState_done) { if (typeof options.onchange === 'function') options.onchange(XMLHttp.readyState, XMLHttp); } else if (typeof options.onfail === 'function') { options.onfail(XMLHttp); } }; } // 若檔案不存在,會 throw。 XMLHttp.send(post_data || null); if (!onload) { // XMLHttp.response blob // XMLHttp.responseText 會把傳回值當字串用 // XMLHttp.responseXML 會把傳回值視為 XMLDocument 物件,而後可用 JavaScript // DOM 相關函式處理 // IE only(?): // XMLHttp.responseBody 以unsigned array格式表示binary data // try{responseBody=(new // VBArray(XMLHttp.responseBody)).toArray();}catch(e){} // http://aspdotnet.cnblogs.com/archive/2005/11/30/287481.html // XMLHttp.responseStream return AdoStream return XMLHttp.responseText; } } catch (e) { library_namespace.error(e); if (typeof options.onfail === 'function') { options.onfail(XMLHttp, e); } else if (onload) { onload(undefined, e); } } } _.get_URL = get_URL; // TODO: 處理 multiple requests function get_URLs() { } // ---------------------------------------------------- var is_nodejs = library_namespace.platform.nodejs; /** * // "file": keyword for "Content-Disposition: file;" {type:'jpg',image:{file:'fn1.jpg'}} // will fetch url first. {type:'jpg',image:{url:'http://host/'}} {type:'jpg',image:{file:'fn1.jpg',type:'image/jpeg'}} // Array: use "Content-Type: multipart/mixed;" {type:'jpg',images:[{file:'fn1.jpg'},{file:'fn2.jpg'}]} {type:'jpg',images:[{file:'fn1.jpg'},{file:'fn2.jpg'}],docs:[{file:'fn1.txt'},{file:'fn2.txt'}]} {type:'jpg',images:[{file:'fn1.jpg',type:'image/jpeg'},{file:'fn1.txt',type:'text/plain'}]} */ // should be CRLF // @see https://tools.ietf.org/html/rfc7578#section-4.1 var form_data_new_line = '\r\n'; function form_data_to_Array(is_slice) { if (this.generated) { return this.generated; } var boundary = '--' + this.boundary + form_data_new_line, // generated raw post data generated = this.generated = [ boundary ], content_length = boundary.length; boundary = form_data_new_line + boundary; this.forEach(function(chunk, index) { if (Array.isArray(chunk)) { // chunk = chunk.to_Array(true); if (!chunk.content_length) { console.log(chunk); throw new Error( // gettext_config:{"id":"the-chunk-do-not-has-regular-.content_length"} 'The chunk do not has regular .content_length!'); } content_length += chunk.content_length; } else { // chunk: {String} or {Buffer} content_length += chunk.length; } generated.push(chunk); if (index < this.length - 1) { generated.push(boundary); content_length += boundary.length; } }, this); if (!(content_length > 0)) { console.log(this); // gettext_config:{"id":"illegal-chunk.content_length"} throw new Error('Illegal chunk.content_length!'); } boundary = form_data_new_line + '--' + this.boundary; if (!is_slice) { boundary += '--'; } generated.push(boundary); content_length += boundary.length; generated.content_length = content_length; // console.log(generated); return generated; } // 選出 data.generated 不包含之 string function give_boundary(data_Array) { function not_includes_in(item) { // console.trace([ typeof item, item ]); return Array.isArray(item) ? item.every(not_includes_in) // item: Should be {String} or {Buffer} : !item.includes(boundary); } var boundary, retry_count = 0; while (retry_count++ < 8) { boundary = (Number.MAX_SAFE_INTEGER * Math.random()) .toString(10 + 26); // console.log('test boundary: [' + boundary + ']'); for (var i = 1; i < boundary.length / 2 | 0; i++) { var slice = boundary.slice(0, i); if (boundary.lastIndexOf(slice) > 0) { boundary = null; break; } } // assert: boundary 不自包含,例如 'aa'自包含'a','asas'自包含'as' if (boundary) { if (not_includes_in(data_Array)) { data_Array.boundary = boundary; return boundary; } } } throw new Error('give_boundary: ' // gettext_config:{"id":"retry-too-many-times"} + 'Retry too many times!'); } var to_form_data_generated = { form_data_generated : true }; // https://github.com/form-data/form-data/blob/master/lib/form_data.js // https://www.w3.org/TR/html401/interact/forms.html#h-17.13.4.2 // https://tools.ietf.org/html/rfc7578 // https://tools.ietf.org/html/rfc2046#section-5.1 // The "multipart" boundary delimiters and header fields are always // represented as 7bit US-ASCII // https://tools.ietf.org/html/rfc2049#appendix-A // http://stackoverflow.com/questions/4238809/example-of-multipart-form-data function to_form_data(parameters, callback, options) { function get_file_object(value, callback, key, slice) { var is_url, MIME_type; if (typeof value === 'string') { is_url = value.includes('://'); } else // else: assert: library_namespace.is_Object(value) if (is_url = value.url) { value = is_url; // is_url = true; } else { // .type: MIME type MIME_type = value.type; // value: file_path value = value.file; } function push_and_callback(MIME_type, content) { // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition var headers = 'Content-Disposition: ' + (slice ? 'file' : 'form-data; name="' + key + '"') + '; filename="' + encodeURIComponent(value) + '"' + "; filename*=UTF-8''" + encodeURIComponent(value) + form_data_new_line; if (MIME_type) { headers += 'Content-Type: ' + MIME_type + form_data_new_line; } if (is_nodejs && Buffer.isBuffer(content)) { headers += 'Content-Transfer-Encoding: binary' + form_data_new_line; } headers += form_data_new_line; var chunk = [ headers, content ]; // 手動設定 Content-Length。 chunk.content_length = headers.length + content.length; // TODO: use stream (slice || root_data).push(chunk); callback(); } if (!is_url) { var content; // read file contents if (is_nodejs) { try { // get {Buffer} content = node_fs.readFileSync(value); } catch (e) { // When we cannot read, it will throw now. // e.g., Error: ENOENT: no such file or directory, open // 'value' } } else { // node.js 之下此方法不能處理 binary data。 content = library_namespace .get_file(value/* , 'binary' */); } if (!content) { library_namespace.error([ 'to_form_data: ', { // gettext_config:{"id":"failed-to-get-file-$1"} T : [ 'Failed to get file: [%1]', value ] } ]); // Skip this one. callback(); return; } if (options && options.file_post_processor) { options.file_post_processor(value, content); } // value: file path → file name value = value.match(/[^\\\/]*$/)[0]; if (!MIME_type) { MIME_type = library_namespace.MIME_of(value); } push_and_callback(MIME_type, content); return; } library_namespace.debug({ // gettext_config:{"id":"fetching-url-$1"} T : [ '自網路取得 URL:%1', value ] }, 1, 'to_form_data'); _.get_URL(value, function(XMLHttp, error) { if (options && options.url_post_processor) { options.url_post_processor(value, XMLHttp, error); } if (error) { library_namespace.error([ 'to_form_data: ', { // gettext_config:{"id":"got-error-when-retrieving-$1-$2"} T : [ 'Got error when retrieving [%1]: %2', // value, localize_error(error) ] } ]); // Skip this one. callback(); return; } // value: url → file name value = value.replace(/[?#].*$/, '') .match(/([^\\\/]*)[\\\/]?$/)[1]; // console.log('-'.repeat(79)); // console.log(value); library_namespace.debug({ // gettext_config:{"id":"got-url-from-the-network-$1-$2-bytes"} T : [ '自網路取得 URL:%1,%2{{PLURAL:%2|位元組}}。', value, XMLHttp.buffer.length ] }, 1, 'to_form_data'); push_and_callback(XMLHttp.type, XMLHttp.buffer); }, 'buffer'); } parameters = library_namespace.Search_parameters(parameters); var root_data = [], keys = Object.keys(parameters), index = 0; root_data.to_Array = form_data_to_Array; // console.log('-'.repeat(79)); // console.log(keys); // 因為在遇到fetch url時需要等待,因此採用async。 function process_next() { if (false) { console.log('-'.repeat(60)); console.log('process_next: ' + index + '/' + keys.length); console.log(root_data); } if (index === keys.length) { // 決定 boundary give_boundary(root_data); // WARNING: 先結束作業: 生成 .to_Array(), // 才能得到 root_data.to_Array().content_length。 root_data.to_Array(); if (false) { console.log('-'.repeat(79)); console.log(root_data); console.log('-'.repeat(79)); console.log(root_data.to_Array().content_length); console.log(root_data.to_Array().join('')); throw 5; } callback(root_data); return; } var key = keys[index++], value = parameters[key]; // console.log(key + ': ' + JSON.stringify(value)); if (Array.isArray(value)) { // assert: is files/urls var slice = [], item_index = 0, // next_item = function() { if (item_index === value.length) { give_boundary(slice); var headers = 'Content-Disposition: form-data; name="' + key + '"' + form_data_new_line + 'Content-Type: multipart/mixed; boundary=' + slice.boundary + form_data_new_line + form_data_new_line; slice = form_data_to_Array.call(slice, true); slice.unshift(headers); slice.content_length += headers.length; root_data.push(slice); process_next(); } else { get_file_object(value[item_index++], next_item,/* key */ undefined, slice); } }; next_item(); return; } if (library_namespace.is_Object(value)) { // assert: is file/url get_file_object(value, process_next, key); return; } // 預防有 null, undefined, {Number}, true 之類。 if (!value || typeof value === 'number' || typeof value === 'boolean') { value = String(value); } var headers; if (is_nodejs && typeof value === 'string') { value = Buffer.from(value, 'utf8'); headers = 'Content-Type: text/plain; charset=UTF-8' + form_data_new_line; } // 非檔案,屬於普通的表單資料。 if (!key) { throw new Error('No key for value: ' + value); } if (!value && value !== '') { // e.g., token=undefined throw new TypeError('Invalid type of ' + key + ': ' + typeof value); } // @see function push_and_callback(MIME_type, content) headers = 'Content-Disposition: form-data; name="' + key + '"' + form_data_new_line + (headers || '') + form_data_new_line; var chunk = [ headers, value ]; // 手動設定 Content-Length。 chunk.content_length = headers.length + value.length; root_data.push(chunk); process_next(); } process_next(); return root_data; } _.to_form_data = to_form_data; // ---------------------------------------------------------------------// /** * 讀取URL by XMLHttpRequest http://jck11.pixnet.net/blog/post/11630232 * 若有多行程或為各URL設定個別XMLHttp之必要,請在一開始便設定deprecated_get_URL.multi_request,並且別再更改。 ** 在此情況下,單一URL仍只能有單一個request! ** 設定 handle_function 須注意程式在等待回應時若無執行其他程式碼將自動中止! 可設定: while(deprecated_get_URL.doing)WScript.Sleep(1); //||timeout arguments f:{ URL:'', // The same origin policy prevents document or script loaded from one origin, from getting or setting properties from a of a document from a different origin.(http://www.mozilla.org/projects/security/components/jssec.html#sameorigin) enc:'UTF-8', // charset: big5, euc-jp,.. fn:(handle_function), // onLoad:function(){}, method:'GET', // POST,.. sendDoc:'text send in POST,..' async:ture/false, // true if want to asynchronous(非同期), false if synchronous(同期的,會直到readyState==4才return) http://jpspan.sourceforge.net/wiki/doku.php?id=javascript:xmlhttprequest:behaviour user:'userName', passwd:'****', // password //TODO: parameters:'~=~&~=~', // {a:1,b:2} header:{contentType:'text/xml'}, contentType:'text/xml', run:true/false, // do eval update:DOMDocument, // use onLoad/onFailed to 加工 return text. onFailed(){throw;} will abort change. interval:\d, decay:\d, // wait decay*interval when no change maxInterval::\d, //insertion:top/bottom,.. onFailed:function(error){this.status;}, // onFailed.apply(XMLHttp,[XMLHttp.status]) onStateChange:function(){}, } handle_function: 自行處理 typeof handle_function=='function': function handle_function(error){..} 代為處理 handle_function=[d_func,0: responseText,1: responseXML]: responseXML: http://msdn2.microsoft.com/en-us/library/ms757878.aspx function d_func(content,headers[,XMLHttp,URL]){ if(headers){ // content,headers各為XMLHttp.responseText內容及XMLHttp.getAllResponseHeaders(),其他皆可由XMLHttp取得。 }else{ // content為error } } e.g., the simplest: [function(c,h){h&&alert(c);}] ) */ // (URL,fn) or flag URL, handle_function handle result, // method,sendDoc,asyncFlag,userName,password function deprecated_get_URL(f) { var _f = arguments.callee; if (typeof _f.XMLHttp === 'object') { // try{_f.XMLHttp.abort();}catch(e){} // 此時可能衝突或lose?! _f.XMLHttp = null; } // 處理 arguments if (!library_namespace.is_Object(f)) a = arguments, f = { URL : f, fn : a[1], method : a[2], sendDoc : a[3] }; if (f.post) f.method = 'POST', f.sendDoc = f.post; if (!f[KEY_URL] || !(_f.XMLHttp = library_namespace.new_XMLHttp(f.enc, !/\.x(?:ht)?ml$/i.test(f[KEY_URL])))) // throw return; // try{_f.XMLHttp.overrideMimeType('text/xml');}catch(e){} if (typeof f.async !== 'boolean') // 設定f.async f.async = f.fn ? true : false; else if (!f.async) f.fn = null; else if (!f.fn) if (typeof _f.HandleStateChange !== 'function' || typeof _f.HandleContent !== 'function') // 沒有能處理的function // throw return; else // =null; f.fn = _f.HandleContent; if (/* typeof _f.multi_request!=='undefined'&& */_f.multi_request) { if (!_f.q) // queue _f.i = {}, _f.q = []; // ** 沒有考慮到 POST 時 URL 相同的情況! _f.i[f[KEY_URL]] = _f.q.length; _f.q.push({ uri : f[KEY_URL], XMLHttp : _f.XMLHttp, func : f.fn, start : _f.startTime = new Date }); } else if (_f.q && typeof _f.clean === 'function') _f.clean(); // for Gecko Error: uncaught exception: Permission denied to call method // XMLHttpRequest.open if (f[KEY_URL].indexOf('://') !== NOT_FOUND && typeof netscape === 'object') if (_f.asked > 2) { _f.clean(f[KEY_URL]); return; } else try { if (typeof _f.asked === 'undefined') { _f.asked = 0; alert('我們需要一點權限來使用 XMLHttpRequest.open。\n* 請勾選記住這項設定的方格。'); } netscape.security.PrivilegeManager // UniversalBrowserAccess .enablePrivilege('UniversalXPConnect'); } catch (e) { _f.asked++; _f.clean(f[KEY_URL]); return; } // if(isNaN(_f.timeout))_f.timeout=300000;//5*60*1000; try { // IE:404會throw error, timeout除了throw error, 還會readystatechange; // Gecko亦會throw error try { _f.XMLHttp.setRequestHeader("Accept-Encoding", "gzip, deflate, br"); } catch (e) { } // Set header so the called script knows that it's an XMLHttpRequest if (false) _f.XMLHttp.setRequestHeader("X-Requested-With", "XMLHttpRequest"); // Set the If-Modified-Since header, if ifModified mode. if (false) _f.XMLHttp.setRequestHeader("If-Modified-Since", "Thu, 01 Jan 1970 00:00:00 GMT"); if (f.method === 'POST' // &&_f.XMLHttp.setRequestHeader ) { // use .getAttribute('method') to get 長度不一定如此 if (false) _f.XMLHttp.setRequestHeader("Content-Length", f.sendDoc.length); // 有些CGI會用Content-Type測試是XMLHttp或是regular form // It may be necessary to specify // "application/x-www-form-urlencoded" or "multipart/form-data" // for posted XML data to be interpreted on the server. _f.XMLHttp.setRequestHeader('Content-Type', Array.isArray(f.fn) && f.fn[1] ? 'text/xml' // application/x-www-form-urlencoded; charset=utf-8 : 'application/x-www-form-urlencoded'); } _f.XMLHttp.abort(); _f.XMLHttp.open(f.method || 'GET', f[KEY_URL], f.async, f.user || null, f.passwd || null); // alert((f.method||'GET')+','+f[KEY_URL]+','+f.async); /** * @see http://www.javaworld.com.tw/jute/post/view?bid=49&id=170177&sty=3&age=0&tpg=1&ppg=1 * 根據 W3C的 XMLHttpRequest 規格書上說,①在呼叫 open * 時,如果readyState是4(Loaded) ②呼叫abort之後 * ③發生其他錯誤,如網路問題,無窮迴圈等等,則會重設所有的值。使用全域的情況就只有第一次可以執行,因為之後的readyState是4,所以onreadystatechange * 放在open之前會被清空,因此,onreadystatechange 必須放在open之後就可以避免這個問題。 * * 每使用一次XMLHttpRequest,不管成功或失敗,都要重設onreadystatechange一次。onreadystatechange * 的初始值是 null * * @see http://www.xulplanet.com/references/objref/XMLHttpRequest.html * After the initial response, all event listeners will be * cleared. Call open() before setting new event listeners. */ if (f.async) { _f.doing = (_f.doing || 0) + 1; _f.XMLHttp.onreadystatechange = typeof f.fn === 'function' // ? f.fn : function(e) { _f.HandleStateChange(e, f[KEY_URL], f.fn); } // ||null ; // 應加 clearTimeout( ) setTimeout('try{deprecated_get_URL.' // + (_f.multi_request ? 'q[' + _f.i[f[KEY_URL]] + ']' : 'XMLHttp') + '.onreadystatechange();}catch(e){}', // 5*60*1000; _f.timeout || 3e5); } _f.XMLHttp.send(f.sendDoc || null); if (!f.fn) { /** * 非async(異步的)能在此就得到 response。Safari and Konqueror cannot * understand the encoding of text files! * * @see http://www.kawa.net/works/js/jkl/parsexml.html */ // responseXML: responseXML.loadXML(text) return _f.XMLHttp.responseText; } } catch (e) { if (typeof f.fn === 'function') f.fn(e); else if (typeof window === 'object') window.status = e.message; return e; } } deprecated_get_URL.timeoutCode = -7732147; /** * agent handle function * * e: object Error, handle_function: function(return text, headers, * XMLHttpRequest object, URL) | [ function, (default|NULL:responseText, * others:responseXML) ] */ deprecated_get_URL.HandleStateChange = function(e, URL, handle_function) { var _t = 0, isOKc, m = deprecated_get_URL.multi_request, _oXMLH; if (m) m = deprecated_get_URL.q[isNaN(URL) ? deprecated_get_URL.i[URL] : URL], _oXMLH = m.XMLHttp, handle_function = m.func, URL = m.uri; else _oXMLH = deprecated_get_URL.XMLHttp; if (Array.isArray(handle_function)) _t = handle_function[1], handle_function = handle_function[0]; if (!handle_function || typeof handle_function !== 'function') { deprecated_get_URL.doing--; deprecated_get_URL.clean(URL); return; } // http://big5.chinaz.com:88/book.chinaz.com/others/web/web/xml/index1/21.htm if (!e) if (typeof _oXMLH === 'object' && _oXMLH) { if (_oXMLH.parseError && _oXMLH/* .responseXML */.parseError.errorCode !== 0) e = _oXMLH.parseError, e = new Error(e.errorCode, e.reason); else if (_oXMLH.readyState === 4) { // only if XMLHttp shows "loaded" // condition is OK? isOKc = _oXMLH.status; isOKc = isOKc >= 200 && isOKc < 300 || isOKc === 304 || !isOKc && (location.protocol === "file:" || location.protocol === "chrome:"); if (handle_function === deprecated_get_URL.HandleContent) // handle_function.apply() handle_function(0, isOKc, _oXMLH, URL); else { // handle_function.apply() handle_function(isOKc ? _t ? _oXMLH.responseXML // JKL.ParseXML: Safari and Konqueror cannot // understand the encoding of text files. : typeof window === 'object' && window.navigator.appVersion .indexOf("KHTML") !== NOT_FOUND && !(e = escape(_oXMLH.responseText)) .indexOf("%u") !== NOT_FOUND // ? e : _oXMLH.responseText : 0, // isOKc ? _oXMLH.getAllResponseHeaders() : 0, _oXMLH, URL); } // URL之protocol==file: // 可能需要重新.loadXML((.responseText+'').replace(/<\?xml[^?]*\?>/,"")) // 用 .responseXML.documentElement 可調用 deprecated_get_URL.doing--; deprecated_get_URL.clean(URL); return; } } else if (new Date - (m ? m.start : deprecated_get_URL.startTime) > deprecated_get_URL.timeout) // timeout & timeout function // http://www.stylusstudio.com/xmldev/199912/post40380.html // _oXMLH.abort(); e = new Error(deprecated_get_URL.timeoutCode, 'Timeout'); // alert(URL+'\n'+_t+'\n'+e+'\n'+_oXMLH.readyState+'\n'+handle_function); if (e) { handle_function(e, 0, _oXMLH, URL); deprecated_get_URL.doing--; deprecated_get_URL.clean(URL); }// handle_function.apply(e,URL); }; /** * agent content handle function
* 有headers時content包含回應,否則content表error */ deprecated_get_URL.HandleContent = function(content, headers, _oXMLHttp, URL) { if (headers) { // _oXMLHttp.getResponseHeader("Content-Length") alert("URL: " + URL + "\nHeaders:\n" + _oXMLHttp.getAllResponseHeaders() + "\n------------------------\nLastModified: " + _oXMLHttp.getResponseHeader("Last-Modified") + "\nResult:\n" + _oXMLHttp.responseText.slice(0, 200));// _oXMLHttp.responseXML.xml } else { // error // test時,可用deprecated_get_URL.XMLHttp.open("HEAD","_URL_",true);,deprecated_get_URL(url,handle_function,'HEAD',true)。 if (content instanceof Error) alert('Error occured!\n' + (typeof e === 'object' && e.number ? e.number + ':' + e.message : e || '')); else if (typeof _oXMLHttp === 'object' && _oXMLHttp) alert((_oXMLHttp.status === 404 ? "URL doesn't exist!" : 'Error occured!') + '\n\nStatus: ' + _oXMLHttp.status + '\n' + _oXMLHttp.statusText); } }; // 在MP模式下清乾淨queue deprecated_get_URL.clean = function(i, force) { // multiple requests if (force || deprecated_get_URL.multi_request) if (!i && isNaN(i)) { if (deprecated_get_URL.q) for (i in deprecated_get_URL.i) try { deprecated_get_URL.q[deprecated_get_URL.i[i]].XMLHttp .abort(); // deprecated_get_URL.q[deprecated_get_URL.i[i]].XMLHttp=null; } catch (e) { } deprecated_get_URL.q = deprecated_get_URL.i // =null = 0; } else if (!isNaN(i) || !isNaN(i = deprecated_get_URL.i[typeof i === 'object' ? i.uri : i])) { try { deprecated_get_URL.q[i].XMLHttp.abort(); } catch (e) { } // deprecated_get_URL.q[i].XMLHttp=0; delete deprecated_get_URL.i[deprecated_get_URL.q[i].uri]; deprecated_get_URL.q[i] = 0; } }; // ↑XMLHttp set ================== // ---------------------------------------------------------------------// // @see https://github.com/request/request var node_url, node_http, node_http2, node_https, // reuse the sockets (keep-alive connection). node_http_agent, node_http2_agent, node_https_agent, // node_zlib; /** * 快速 merge cookie: 只檢查若沒有重複的 key,則直接加入。不檢查 path 也不處理 expires, domain, * secure。
* 為增加效率,不檢查 agent.last_cookie 本身之重複的 cookie。 * * TODO: create class Cookie, Cookie.prototype.merge(), * Cookie.prototype.clone() * * @param {Object}agent * node_http_agent / node_https_agent * @param {Array}cookie * new cookie to merge * * @returns {Object}agent.last_cookie * * @inner */ function merge_cookie(agent, cookie) { // 初始化 initialization + 正規化 normalization var last_cookie = agent.last_cookie; if (!Array.isArray(last_cookie)) { last_cookie = agent.last_cookie = agent.last_cookie ? [ agent.last_cookie ] : []; } if (!cookie) { cookie = []; } else if (typeof cookie === 'string') { cookie = cookie.split(';'); } // assert: Array.isArray(cookie) // console.log(agent); // console.log(last_cookie.cookie_hash); // console.trace(cookie); // cookie_index_of[key] = index of last_cookie var cookie_index_of = last_cookie.cookie_index_of; if (!cookie_index_of) { if (last_cookie.length > 0) { // regenerate agent.last_cookie delete agent.last_cookie; last_cookie = merge_cookie(agent, last_cookie); // assert: last_cookie === agent.last_cookie } else { last_cookie.cookie_index_of = Object.create(null); last_cookie.cookie_hash = Object.create(null); } cookie_index_of = last_cookie.cookie_index_of; } var cookie_hash = last_cookie.cookie_hash; // assert: !!cookie_hash === true cookie.forEach(function for_each_cookie_piece(piece) { piece = piece.trim(); if (!piece) return; // [ cookie value without path / domain / expires, // key, value, extra ] var matched = piece.match(/^([^=;]+)(?:=([^;]+))?(.*)$/); library_namespace.debug('last_cookie: ' + last_cookie, 3, 'merge_cookie'); // console.log(matched); var key, value; if (matched) { key = matched[1]; value = matched[2]; } else { library_namespace.warn([ 'merge_cookie: ', { // gettext_config:{"id":"invalid-cookie"} T : 'Invalid cookie?' }, ' [' + piece + ']' ]); // treat cookie piece as key key = piece; value = ''; } if (!key) return; cookie_hash[key] = value; if (key in cookie_index_of) { // assert: (key in cookie_hash) === true library_namespace.debug([ { // gettext_config:{"id":"duplicate-cookie-name!-the-later-newcomer-will-prevail"} T : 'cookie 名稱重複!以後來/新出現者為準。' }, ' [' + last_cookie[cookie_index_of[key]] // + ']→[' + piece + ']' ], 3, 'merge_cookie'); // remove duplicate cookie: 直接取代。 last_cookie[cookie_index_of[key]] = piece; } else { // assert: (key in cookie_hash) === false library_namespace.debug([ { T : '正常情況。登記已存在之 cookie。' } ], 3, 'merge_cookie'); // console.trace(matched); cookie_index_of[key] = last_cookie.length; last_cookie.push(piece); } }); // console.trace(cookie_hash); // console.trace(last_cookie); library_namespace.debug('array: ' + JSON.stringify(last_cookie), 3, 'merge_cookie'); library_namespace.debug('hash: ' + JSON.stringify(cookie_hash), 3, 'merge_cookie'); return last_cookie; } _.merge_cookie = merge_cookie; function set_cookie_to_URL_object(URL_options_to_fetch, agent) { // console.trace('agent.last_cookie:'); // console.log(agent.last_cookie); if (agent.last_cookie) { // 使用 cookie library_namespace.debug('agent.last_cookie: ' + JSON.stringify(agent.last_cookie), 3, 'set_cookie_to_URL_object'); library_namespace.debug('agent.last_cookie.cookie_hash: ' + JSON.stringify(agent.last_cookie.cookie_hash), 3, 'set_cookie_to_URL_object'); var cookie = (URL_options_to_fetch.headers.Cookie ? URL_options_to_fetch.headers.Cookie + ';' : '') // cookie is Array @ Wikipedia + (Array.isArray(agent.last_cookie) ? agent.last_cookie // 去掉 expires=...; path=/; domain=...; HttpOnly // 這個動作不做也可以,不影響結果。 .map(function(cookie) { return cookie.replace(/;.*/, ''); }).join('; ') : agent.last_cookie); if (cookie) { URL_options_to_fetch.headers.Cookie = cookie; } else { delete URL_options_to_fetch.headers.Cookie; } } library_namespace.debug('Set cookie: ' + JSON.stringify(URL_options_to_fetch.headers.Cookie), 3, 'set_cookie_to_URL_object'); library_namespace.debug('Set protocol: ' + URL_options_to_fetch.protocol, 3, 'set_cookie_to_URL_object'); library_namespace.debug('Set headers: ' + JSON.stringify(URL_options_to_fetch.headers), 3, 'set_cookie_to_URL_object'); } // ---------------------------------------------------------------------// // 正處理中之 connections var get_URL_node_connections = 0, // 所有 requests get_URL_node_requests = 0; // 強制使用POST傳送。 var FORCE_POST = { FORCE_POST : true }; var ERROR_BAD_STSTUS = 'BAD STATUS'; var KEY_not_native = library_namespace.env.not_native_keyword; var has_native_URL = typeof URL === "function" && !URL[KEY_not_native]; /** * 讀取 URL via node http/https。
* assert: arguments 必須與 get_URL() 相容! * * @param {String|Object}URL_to_fetch * 欲請求之目的 URL or options * @param {Function}[onload] * callback when successful loaded. For failure handling, using * options.onfail(error); * @param {String}[charset] * character encoding of HTML web page. e.g., 'UTF-8', big5, * euc-jp,.. * @param {String|Object}[post_data] * text data to send when method is POST * @param {Object}[options] * 附加參數/設定選擇性/特殊功能與選項 * * @see https://nodejs.org/api/http.html#http_http_request_options_callback * https://nodejs.org/api/https.html#https_https_request_options_callback * * @since 2015/1/13 23:23:38 */ function get_URL_node(URL_to_fetch, onload, charset, post_data, options) { if (!URL_to_fetch) { onload(undefined, new SyntaxError('No URL input.')); return; } get_URL_node_requests++; if (get_URL_node_connections >= get_URL_node.connects_limit) { library_namespace.debug({ // gettext_config:{"id":"waiting-$1-$2-connections-$3"} T : [ 'Waiting %1/%2 {{PLURAL:%1|connection|connections}}: %3', // 避免同時開過多 connections 的機制。 get_URL_node_connections, get_URL_node_requests, String(URL_to_fetch) ] }, 3, 'get_URL_node'); var _arguments = arguments; setTimeout(function() { get_URL_node_requests--; get_URL_node.apply(null, _arguments); }, 500); return; } // 進入 request 程序 get_URL_node_connections++; // 前導作業。 if (library_namespace.is_Object(charset)) { post_data = charset; charset = null; } // 正規化並提供可隨意改變的同內容參數,以避免修改或覆蓋附加參數。 options = library_namespace.new_options(options); // console.log('-'.repeat(79)); // console.log(JSON.stringify(options)); // console.log(options.form_data); if (options.form_data && options.form_data !== to_form_data_generated) { // console.trace(options); // TODO: charset for post_data to_form_data(post_data, function(data) { // console.log(data.toString().slice(0,800)); // console.log('>> ' + data.toString().slice(-200)); options.form_data = to_form_data_generated; get_URL_node(URL_to_fetch, onload, charset, data, options); }, options.form_data); return; } // ------------------------------------------------------ if (library_namespace.is_Object(URL_to_fetch) && URL_to_fetch[KEY_URL]) { Object.assign(options, URL_to_fetch); // 注意: options.onload 另有用途! // https://xhr.spec.whatwg.org/#handler-xhr-onloadstart // onload = options.onload || onload; post_data = options.post || post_data; charset = options.charset || charset; URL_to_fetch = options[KEY_URL]; } // 不改變 options。 var agent = options.agent; if (typeof URL_to_fetch === 'string' && URL_to_fetch.startsWith('//')) { // 處理 '//domain.org/path' 的情況。 URL_to_fetch = (agent && agent.protocol || 'https:') + URL_to_fetch; } var URL_options_to_fetch = normalize_URL_to_fetch(URL_to_fetch, charset, options); // assert: library_namespace.is_URI(URL_options_to_fetch) // console.trace([ URL_to_fetch, URL_options_to_fetch ]); if (typeof onload === 'object') { library_namespace.debug( 'Trying to JSONP, insert page, need callback.', 3, 'get_URL_node'); // library_namespace.run(URL_options_to_fetch); for ( var callback_param in onload) { if (callback_param && typeof onload[callback_param] === 'function') { // 模擬 callback。 // callback_param: callback parameter URL_options_to_fetch.search_params[callback_param] = 'cb'; onload = onload[callback_param]; break; } } } // assert: 自此開始不會改變 URL,也不會中途 exit 本函數。 if (post_data && !options.form_data) { if (library_namespace.is_Object(post_data) && options.headers && typeof options.headers['Content-Type'] === 'string' && options.headers['Content-Type'] .includes('application/json')) { post_data = JSON.stringify(post_data) || FORCE_POST; } else { post_data = library_namespace.Search_parameters(post_data) .toString(charset) || FORCE_POST; } } if (!onload && typeof options.onchange === 'function') { onload = function() { options.onchange(readyState_done); }; } if (options.async === false && onload || typeof onload !== 'function') { onload = false; } // console.trace(URL_options_to_fetch); // node_http.request(options): options needs a .path // https://nodejs.org/dist/latest/docs/api/http.html#http_http_request_options_callback URL_options_to_fetch.path = URL_options_to_fetch.pathname + URL_options_to_fetch.search; var URL_is_https = /^https:?$/i.test(URL_options_to_fetch.protocol); // console.trace([ URL_to_fetch, URL_options_to_fetch.toString() ]); URL_to_fetch = URL_options_to_fetch.toString(); // assert: {String}URL_to_fetch, // library_namespace.is_URI(URL_options_to_fetch) /** * https://stackoverflow.com/questions/53593182/client-network-socket-disconnected-before-secure-tls-connection-was-established * mh160.js 必須使用 request,https-proxy-agent 才能正常工作 TODO: * socks-proxy-agent * * // https://techcult.com/free-proxy-software-for-windows-10/#1_Ultrasurf > SET http_proxy=http://127.0.0.1:9666 > node u17.js 镇魂街 */ /** * // http://anonproxyserver.sourceforge.net/ // https://www.proxynova.com/proxy-server-list/country-tw/ var http = require("http"); var options = { host: "211.22.233.69", port: 3128, path: "http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout", //method: 'GET', headers: { Host: "dict.revised.moe.edu.tw" } }; var request=http.request(options, function(response) { console.log(response.statusCode); console.log(response.headers); var data = [], length = 0; response.on('data', function(chunk) {length += chunk.length;data.push(chunk);}); response.on('end', function() {data = Buffer.concat(data, length);console.log(data+'')}); }); request.end(); require('./work_crawler_loader.js'); var PROXY='localhost:8080'; CeL.get_URL('https://zh.wikipedia.org/wiki/Special:%E6%9C%80%E8%BF%91%E6%9B%B4%E6%94%B9',function(X){console.log(X.responseText)},null,null,{proxy:PROXY}); CeL.get_URL('https://zh.wikipedia.org/wiki/Special:%E6%9C%80%E8%BF%91%E6%9B%B4%E6%94%B9',function(X){console.log(X.responseText)}); CeL.get_URL('http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout',function(X){console.log(X.responseText)},null,null,{proxy:PROXY}); CeL.get_URL('http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi?ccd=9gW4am&o=e0&sec=sec11&option=linkout001&actice=layout',function(X){console.log(X.responseText)}); // TODO: test agent, cookie */ var proxy_original_agent, // using_proxy_server proxy_server = options.proxy // https://curl.haxx.se/docs/manpage.html // https://superuser.com/questions/876100/https-proxy-vs-https-proxy // https://docs.oracle.com/cd/E56344_01/html/E54018/gmgas.html // https://stackoverflow.com/questions/32824819/difference-between-http-proxy-https-proxy-and-proxy || URL_is_https && process.env.HTTPS_PROXY // `SET http_proxy=http://127.0.0.1:8080` || process.env.http_proxy; if (!proxy_server || !(proxy_server = library_namespace.URI(proxy_server))) { ; } else if (URL_is_https) { library_namespace.debug({ // gettext_config:{"id":"using-https-proxy-to-get-url-$1"} T : [ 'Using HTTPS proxy to get URL: %1', URL_to_fetch ] }, 2, 'get_URL_node'); // ... just add the special agent: proxy_original_agent = proxy_server.agent = agent; agent = new HttpsProxyAgent(proxy_server); // 複製必要的舊屬性。 if (proxy_original_agent && proxy_original_agent.last_cookie) { // 複製原agent的cookie設定。 @see merge_cookie() agent.last_cookie = proxy_original_agent.last_cookie; } // https://github.com/TooTallNate/node-https-proxy-agent/blob/master/index.js // ALPN is supported by Node.js >= v5. // attempt to negotiate http/1.1 for proxy servers that support // http/2 if (!('ALPNProtocols' in URL_options_to_fetch)) { URL_options_to_fetch.ALPNProtocols = [ 'http 1.1' ]; } } else { library_namespace.debug({ // gettext_config:{"id":"using-http-proxy-to-get-url-$1"} T : [ 'Using HTTP proxy to get URL: %1', URL_to_fetch ] }, 2, 'get_URL_node'); // https://www.proxynova.com/proxy-server-list/country-tw/ // proxy_server.URL_to_fetch = URL_to_fetch; // 代理伺服器 using proxy server // https://stackoverflow.com/questions/3862813/how-can-i-use-an-http-proxy-with-node-js-http-client // https://cnodejs.org/topic/530f41e75adfcd9c0f1c8c16 URL_options_to_fetch = get_proxy_URL(proxy_server, URL_options_to_fetch, URL_to_fetch); } if (!URL_options_to_fetch.protocol) { URL_options_to_fetch.protocol = agent && agent.protocol // 直接設定。 default: https:// || (proxy_server ? 'http:' : 'https:'); URL_is_https = URL_options_to_fetch.protocol === 'https:'; } if (agent) { library_namespace.debug({ // gettext_config:{"id":"using-new-agent"} T : agent === true ? '使用新 agent。' // gettext_config:{"id":"using-custom-agent"} : '使用自定義 agent。' }, 6, 'get_URL_node'); if (agent === true) { // use new agent. options.agent = agent = URL_is_https ? new node_https.Agent : new node_http.Agent; } else if (agent.protocol // agent.protocol 可能是 undefined。 && agent.protocol !== URL_options_to_fetch.protocol) { var message = { // gettext_config:{"id":"the-custom-agent-is-different-from-the-url-and-will-try-to-adopt-the-conforming-agreement-$1"} T : [ '自定義 agent 與 URL 之協定不同,將嘗試採用符合的協定:%1', // agent.protocol + ' !== ' + URL_options_to_fetch.protocol ] }; if (options.no_protocol_warn) { library_namespace.debug(message, 3, 'get_URL_node'); } else { library_namespace.warn([ 'get_URL_node: ', message ]); } // use new agent. // assert: options.agent === agent agent = URL_is_https ? new node_https.Agent : new node_http.Agent; // 複製必要的舊屬性。 if (options.agent.last_cookie) { // 複製原agent的cookie設定。 @see merge_cookie() agent.last_cookie = options.agent.last_cookie; } } } else { // gettext_config:{"id":"using-generic-agent"} library_namespace.debug('採用泛用的 agent。', 6, 'get_URL_node'); agent = URL_is_https ? node_https_agent : node_http_agent; } // console.log([ options.cookie, agent.last_cookie ]); if (options.cookie && !agent.last_cookie) { library_namespace.debug({ // gettext_config:{"id":"reset-the-cookie-to-$1"} T : [ '重新設定 cookie 成:%1', options.cookie ] }, 3, 'get_URL_node'); agent.last_cookie = options.cookie; } var request, finished, // result_Object 模擬 XMLHttp response。 result_Object = { // node_agent : agent, // .url @ fetch() // url : URL_to_fetch, // https://developer.mozilla.org/zh-TW/docs/Web/API/Response // .useFinalURL @ fetch() // useFinalURL : URL_to_fetch, // 因為可能 redirecting 過,這邊列出的才是最終 URL。 // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/responseURL responseURL : URL_to_fetch }, // assert: 必定從 _onfail 或 _onload 作結,以確保會註銷登記。 // 本函數unregister()應該放在所有本執行緒會執行到onload的程式碼中。 unregister = function() { /** * @see http://stackoverflow.com/questions/24667122/http-request-timeout-callback-in-node-js * * sometimes both timeout callback and error callback will be called * (the error inside the error callback is ECONNRESET - connection * reset) * * there is a possibilities that it fires on('response', * function(response)) callback altogether */ if (finished) { return true; } // 註銷登記。 finished = true; get_URL_node_requests--; get_URL_node_connections--; if (timeout_id) { library_namespace.debug('clear timeout ' + time_message(timeout) + ' [' + URL_to_fetch + ']', 3, 'get_URL_node'); // console.trace('clear timeout ' + URL_to_fetch); clearTimeout(timeout_id); } }, // on failed _onfail = function(error) { if (unregister()) { // console.log('exit: 預防 timeout 時重複執行。'); return; } // console.trace([ options.error_count, options.error_retry ]); // 連線逾期/失敗時再重新取得頁面之重試次數。 if (options.error_retry >= 1 // 例如當遇到 404 或 502 時,再多嘗試一下。 && !(options.error_retry <= options.error_count)) { if (!options.get_URL_cloned) { // 不動到原來的 options。 options = Object.clone(options); options.get_URL_cloned = true; } if (options.error_count >= 1) { options.error_count++; } else { options.error_count = 1; } options[KEY_URL] = URL_to_fetch; // Failed to get [' + URL_to_fetch + ']. library_namespace.log([ 'get_URL_node: ', { // gettext_config:{"id":"retry-$1-$2-$3"} T : [ 'Retry %1/%2: %3', options.error_count, // options.error_retry, localize_error(error) ] } ]); // console.error(error); // library_namespace.set_debug(3); get_URL_node(options, onload, charset, post_data); // console.trace(options); return; } if (typeof options.onfail === 'function') { options.onfail.call(result_Object, error); return; } if (!options.no_warning // 應已在 _ontimeout 出過警告訊息。 && error.code !== 'TIMEOUT') { if (error.code === 'ENOTFOUND') { library_namespace.error([ 'get_URL_node: ', { // gettext_config:{"id":"url-not-found-$1"} T : [ 'URL not found: [%1]', URL_to_fetch ] } ]); } else if (error.code === 'EPROTO' && require('tls').DEFAULT_MIN_VERSION === 'TLSv1.2' && library_namespace.platform('node', 12)) { library_namespace.error([ 'get_URL_node: ', { T : // gettext_config:{"id":"node.js-v12-and-later-versions-disable-tls-v1.0-and-v1.1-by-default"} 'Node.js v12 and later versions disable TLS v1.0 and v1.1 by default.' }, { T : // gettext_config:{"id":"please-set-tls.default_min_version-=-tlsv1-first"} 'Please set tls.DEFAULT_MIN_VERSION = "TLSv1" first!' }, ' [' + URL_to_fetch + ']' ]); /** * To solve: get_URL_node: Retry 1/4: Error: write EPROTO 14180:error:1425F102:SSL routines:ssl_choose_client_version:unsupported protocol:c:\ws\deps\openssl\openssl\ssl\statem\statem_lib.c:1922: require('tls').DEFAULT_MIN_VERSION = 'TLSv1'; */ } else { library_namespace.error([ 'get_URL_node: ', { // gettext_config:{"id":"got-error-when-retrieving-$1-$2"} T : [ 'Got error when retrieving [%1]: %2', // URL_to_fetch, localize_error(error) ] } ]); // 這裡用太多並列處理,會造成 error.code "EMFILE"。 // console.error(error); // console.error(options); } } // 在出現錯誤時,將 onload 當作 callback。並要確保 {Object}response // 因此應該要先檢查 error 再處理 response typeof onload === 'function' && onload(result_Object, error); }, // on success _onload = function(response) { // response object: Class: http.IncomingMessage // 在這邊不過剛開始從伺服器得到資料,因此還不可執行unregister(),否則依然可能遇到timeout。 if (finished) { return; } if (options.onresponse) { options.onresponse(response); } // https://xhr.spec.whatwg.org/#handler-xhr-onloadstart if (false && options.onloadstart) { options.onloadstart(); } // 若原先有agent,應該合併到原先的agent,而非可能為暫時性/泛用的agent。 merge_cookie(options.agent || agent, response.headers['set-cookie']); // 先合併完cookie之後才能轉址,否則會漏失掉須設定的cookie。 if (false && response.complete && (response.statusCode / 100 | 0) !== 2) { console.log('response:'); console.log(response); } if ((response.statusCode / 100 | 0) === 3 && response.headers.location && response.headers.location !== URL_to_fetch && !options.no_redirect) { if (unregister()) { // 預防 timeout 時重複執行。 return; } try { // request.abort(); request.destroy(); } catch (e) { } // e.g., 301 if (!options.get_URL_cloned) { // 不動到原來的 options。 options = Object.clone(options); options.get_URL_cloned = true; } options[KEY_URL] = new URL(response.headers.location, URL_to_fetch); library_namespace.debug({ // gettext_config:{"id":"$1-redirecting-to-$2-←-$3"} T : [ '%1 Redirecting to [%2] ← [%3]', response.statusCode, options[KEY_URL], URL_to_fetch ] }, 1, 'get_URL_node'); get_URL_node(options, onload, charset, // 重新導向的時候亦傳送 post data。For wiki API // e.g., https://github.com/kanasimi/CeJS/issues/12 // 但piaotian.js搜索作品名稱直接302導向時,加入post_data會產生 405 Not Allowed response.statusCode != 302 && post_data); return; } library_namespace.debug({ // gettext_config:{"id":"response-headers-$1"} T : [ 'response HEADERS: %1', // JSON.stringify(response.headers) ] }, 4, 'get_URL_node._onload'); // 模擬 Response of fetch() // https://developer.mozilla.org/zh-TW/docs/Web/API/Response Object.assign(result_Object, { redirected : !!options.get_URL_cloned, // {Number}response.statusCode // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/status status : response.statusCode, statusText : response.statusMessage, // XMLHttp.headers['content-type']==='text/html; charset=utf-8' headers : response.headers }); // 在有 options.onfail 時僅 .debug()。但這並沒啥條理... if (options.onfail || (response.statusCode / 100 | 0) === 2) { library_namespace.debug({ // gettext_config:{"id":"http-status-code-$1-$2"} T : [ 'HTTP status code: %1 %2', response.statusCode, URL_to_fetch ] }, 2, 'get_URL_node'); } else if (!options.no_warning) { library_namespace.warn([ 'get_URL_node: ', { // gettext_config:{"id":"exception-http-status-code-$1-$2"} T : [ 'Exception HTTP status code %1: %2', // response.statusCode, URL_to_fetch ] } ]); // console.trace(response); } // node.js會自動把headers轉成小寫。 // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition // 在503之類的情況下。可能沒"Content-Type:"。這時 response 將無.type。 if (response.headers['content-type']) { // MIME type, media-type: XMLHttp.type result_Object.type = response.headers['content-type'] // charset: XMLHttp.charset .replace(/;(.*)$/, function($0, $1) { var matched = $1.match(/[; ]charset=([^;]+)/i); if (matched) { result_Object.charset = matched[1].trim(); } return ''; }).trim(); } if (response.headers['content-disposition']) { // 從 Content-Disposition 中抽取出檔名。 // ext-value = charset "'" [ language ] "'" value-chars var matched = response.headers['content-disposition'] .match(/ filename\*\s*=\s*([^';]+)'([^';]*)'([^';]+)/); if (matched) { matched = matched[3]; } else if (matched = response.headers['content-disposition'] .match(/ filename\s*=\s*([^';]+)/)) { matched = matched[1]; } if (matched && (matched = matched.trim())) { matched = (matched.match(/^"(.*)"$/) || matched.match(/^'(.*)'$/) || [ , matched ])[1]; if (false) { console.log([ matched, result_Object.charset, charset, !!library_namespace.decode_URI_component ]); } try { if (!/%[\da-f]{2}/.test(matched)) { // 有一些網站經過這個轉換似乎就能夠獲得正確的檔案名稱。 matched = escape(matched); } if (false && (result_Object.charset || charset) && library_namespace.decode_URI_component) { // 現在沒有實例需要用到這個部分。 matched = library_namespace.decode_URI_component( matched, result_Object.charset || charset); } else { matched = decodeURIComponent(matched); // 有的時候還需要這一項。 // matched = unescape(matched); } library_namespace.debug({ // gettext_config:{"id":"file-name-$1"} T : [ '檔案名稱:%1', matched ] }, 3, 'get_URL_node'); } catch (e) { // TODO: handle exception } result_Object.filename = library_namespace .to_file_name(matched); } } // listener must be a function if (typeof onload !== 'function' // && !options.write_to && !options.write_to_directory) { // 照理unregister()應該放這邊,但如此速度過慢。因此改放在 _onload 一開始。 unregister(); library_namespace.warn([ 'get_URL_node: ', { // gettext_config:{"id":"got-url-$1-but-there-is-no-listener"} T : [ 'Got URL [%1], but there is no listener!', // URL_to_fetch ] } ]); // console.log(response); return; } library_namespace.debug({ // gettext_config:{"id":"waiting-to-receive-data-transferred-back-from-url-$1"} T : [ '等待接收從網址 [%1] 傳輸回的資料……', URL_to_fetch ] }, 3, 'get_URL_node'); var flow_encoding = response.headers['content-encoding']; flow_encoding = flow_encoding && flow_encoding.trim().toLowerCase(); if (false) { var pipe = response; if (flow_encoding === 'gzip') pipe = pipe.pipe(node_zlib.createGunzip()); else if (flow_encoding === 'deflate') pipe = pipe.pipe(node_zlib.createInflate()); pipe = pipe.pipe(node_fs.createWriteStream(file_path)); } // 準備開始接收資料 // options.ondatastart: 非正規標準 if (options.ondatastart) { options.ondatastart(response); } // options.onload https://xhr.spec.whatwg.org/#handler-xhr-onload /** {Array} [ {Buffer}, {Buffer}, ... ] */ var data = [], length = 0, // total_size https://xhr.spec.whatwg.org/#progressevent total_length = +response.headers['content-length'], lengthComputable = total_length >= 0; response.on('data', function(chunk) { // {Buffer}chunk length += chunk.length; var message = [ (options.write_to ? options.write_to + ': ' : '') // + chunk.length + '/' + length + (total_length ? '/' + total_length : '') + ' bytes (' // 00% of 0.00MiB + (total_length ? (100 * length / total_length | 0) + '%, ' : '') + (length / 1.024 / (/* Date.now() */(new Date) .getTime() - start_time)).toFixed(2) + ' KiB/s)' ]; message.push(': ' + URL_to_fetch); library_namespace.debug('receive BODY.length: ' + message.join(''), 4, 'get_URL_node'); if (options.show_progress && length !== total_length) { if (!(options.show_progress > 1)) { message.pop(); } library_namespace.log_temporary(message.join('')); } if (length > options.MAX_BUFFER_SIZE) { if (data) data = null; } else { data.push(chunk); } // node_fs.appendFileSync('get_URL_node.data', chunk); if (options.ondata) { // 注意: 這邊的 chunk 可能是 gzip 之後的資料! options.ondata(chunk); } // https://xhr.spec.whatwg.org/#handler-xhr-onprogress if (false && options.onprogress) { options.onprogress(lengthComputable, length, total_length); } }); // https://iojs.org/api/http.html#http_http_request_options_callback response.on('end', function() { library_namespace.debug('end(): [' + response.statusCode + '] ' + URL_to_fetch, 2, 'get_URL_node'); // 照理應該放這邊,但如此速度過慢。因此改放在 _onload 一開始。 // unregister(); options.onend && options.onend(); // console.log('No more data in response: ' + URL_to_fetch); // it is faster to provide the length explicitly. data = data && Buffer.concat(data, length); // console.log(data.slice(0, 200)); // console.log(data.slice(0, 200).toString()); if (proxy_original_agent) { // recover properties proxy_original_agent.last_cookie = agent.last_cookie; } // 基本檢測。 if ((response.statusCode / 100 | 0) !== 2 // 例如當遇到 404 或 502 時,再多嘗試一下。 && options.error_retry >= 1 // 本條件參考 _onfail。 && !(options.error_retry <= options.error_count)) { _onfail(ERROR_BAD_STSTUS); return; } if (!data) { ; } else if (options.verify) { // test: invalid content type if (typeof options.verify === 'function') { if (!options.verify(data)) { _onfail('INVALID'); return; } } else { // assert: CeL.application.storage.file included // e.g., options.verify === 'png' var file_type = library_namespace.file_type(data, options.verify); if (file_type.verified === false) { _onfail('Invalid ' + options.verify); return; } } } else if (data.length === 0) { // 若是容許空內容,應該特別指定 options.allow_empty。 if (!options.allow_empty) { _onfail('EMPTY'); return; } } // https://github.com/nodejs/node/blob/master/doc/api/zlib.md#compressing-http-requests-and-responses // https://nodejs.org/docs/latest/api/zlib.html // https://gist.github.com/narqo/5265413 // https://github.com/request/request/blob/master/request.js // http://stackoverflow.com/questions/8880741/node-js-easy-http-requests-with-gzip-deflate-compression // http://nickfishman.com/post/49533681471/nodejs-http-requests-with-gzip-deflate-compression if (flow_encoding && data) { library_namespace.debug('content-encoding: ' + flow_encoding, 5, 'get_URL_node'); switch (flow_encoding) { case 'gzip': library_namespace.debug('gunzip ' + data.length + ' bytes data...', 2, 'get_URL_node'); /** * 可能因為呼叫到舊版library,於此有時會出現 "TypeError: Object # has no method 'gunzipSync'" 有時會有 Error: unexpected end of file */ try { data = node_zlib.gunzipSync(data); } catch (error) { library_namespace.error( // get_URL_node: Error: node_zlib.gunzipSync(): // Error: unexpected end of file [http://...] 'get_URL_node: Error: node_zlib.gunzipSync(): ' + localize_error(error) + ' [' + URL_to_fetch + ']'); if (false) { console.log(error); console.log(URL_options_to_fetch); console.log(node_zlib); console.log(data); console.trace( // 'get_URL_node: Error: node_zlib.gunzipSync()'); console.error(error.stack); } // Release memory. 釋放被占用的記憶體. data = null; _onfail(error); return; } break; case 'deflate': library_namespace.debug('deflate data ' + data.length + ' bytes...', 2, 'get_URL_node'); data = node_zlib.deflateSync(data); break; case 'br': library_namespace.debug('brotli data ' + data.length + ' bytes...', 2, 'get_URL_node'); data = node_zlib.brotliDecompressSync(data); break; default: library_namespace.warn([ 'get_URL_node: ', { // gettext_config:{"id":"unknown-http-compression-method-$1"} T : [ 'Unknown HTTP compression method: [%1]', // flow_encoding ] } ]); break; } } // ------------------------------ if (data && response.statusCode === 403 // && data.toString().includes('Cloudflare') // && data.toString().includes(' id="captcha-bypass"')) { // console.log(data.toString()); library_namespace.error([ 'get_URL_node: ', { // https://github.com/Anorov/cloudflare-scrape T : // TODO: https://github.com/codemanki/cloudscraper // gettext_config:{"id":"you-need-to-bypass-the-ddos-protection-by-cloudflare"} 'You need to bypass the DDoS protection by Cloudflare!' } ]); } // ------------------------------ // setup data of result_Object result_Object.buffer = data; // https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest/response result_Object.response = data; // non-standard 非標準: 設定 charset = 'buffer' 的話,將回傳 {Buffer}。 if (data && charset !== 'buffer') { // 未設定 charset 的話,default charset: UTF-8. // buffer.toString(null) will throw! data = data.toString(charset || undefined/* || 'utf8' */); } if (library_namespace.is_debug(4)) { library_namespace.debug( // 'BODY: ' + data, 1, 'get_URL_node'); } // result_Object模擬 XMLHttp。 result_Object.responseText = data; // ------------------------------ if ((response.statusCode / 100 | 0) !== 2) { // console.trace(data); // ssert: options.error_retry >= 1 ? 最後一次 error // : BAD STATUS and get something in `this.response` _onfail(ERROR_BAD_STSTUS); return; } if (unregister()) { // 預防 timeout 時重複執行。 return; } // assert: 執行至此表示成功取得資料、沒有錯誤, // 開始正常運作至結尾,不會再有中途 return。 // TODO: 確保資料完整,例如檢查結尾碼。 // .save_to if (data && (options.write_to || options.write_to_directory)) { var file_path = options.write_to // save to: 設定寫入目標。 || (options.write_to_directory // + library_namespace.env.path_separator // + library_namespace.to_file_name( // URL_to_fetch.replace(/#.*/, '').replace( /[\\\/:*?"<>|]/g, '_'))) // 避免 Error: ENAMETOOLONG: name too long .slice(0, 256); if (!options.no_warning) { library_namespace.info([ 'get_URL_node: ', { T : [ // gettext_config:{"id":"write-$2-bytes-to-file-$1-$3"} 'Write %2 {{PLURAL:%2|byte|bytes}} to file [%1]: %3' // , file_path, data.length, URL_to_fetch ] } ]); } try { var fd = node_fs.openSync(file_path, 'w'); // TODO: use response.pipe(write_stream); // @see GitHub.updater.node.js node_fs.writeSync(fd, data, 0, data.length, null); node_fs.closeSync(fd); // set file modify date // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date // https://tools.ietf.org/html/rfc7231#section-7.1.1.2 if (response.headers['date']) { try { // The "Date" header field represents the date // and time at which the message was originated // fs.utimesSync(path, atime, mtime) // atime: the last time this file was accessed node_fs.utimesSync(file_path, new Date, // mtime: the last time this file was modified response.headers['date']); } catch (e) { // TODO: handle exception } } } catch (e) { library_namespace.error([ 'get_URL_node: ', { T : [ // gettext_config:{"id":"failed-to-write-$2-bytes-to-$1-$3"} 'Failed to write %2 {{PLURAL:%2|byte|bytes}} to [%1]: %3' // , file_path, data.length, URL_to_fetch ] } ]); console.error(e); } } if (typeof options.content_processor === 'function') { options.content_processor( // ({Buffer}contains, URL, status) data, URL_to_fetch, response.statusCode); } if (typeof options.check_reget === 'function' // check_reget(XMLHttp) && options.check_reget(result_Object, options)) { options[KEY_URL] = URL_to_fetch; get_URL_node(options, onload, charset, post_data); return; } if (typeof onload === 'function') { onload(result_Object, !data && !options.allow_empty); } // Release memory. 釋放被占用的記憶體. data = null; // node_fs.appendFileSync('get_URL_node.data', '\n'); }); }; // -------------------------------------------------------------------- // console.trace([ options.headers, URL_options_to_fetch.headers ]); // https://fetch.spec.whatwg.org/#forbidden-header-name // 必要的 headers: User-Agent, Accept-Language。其他是為了模擬得更真實點。 URL_options_to_fetch.headers = Object.assign({ // 網站的主機名稱。 Host : URL_options_to_fetch.host, // User Agent 'User-Agent' : get_URL_node.default_user_agent, // https://developer.mozilla.org/zh-CN/docs/Glossary/Quality_values Accept : 'text/html,application/xhtml+xml,application/xml;q=0.9' // 少了 '*/*' CrossRef API 會回應 406 "No acceptable resource available." + ',image/avif' + ',image/webp,image/apng,*/*;q=0.8' + ',application/signed-exchange;v=b3;q=0.9', // Accept : 'application/json, text/plain, */*', // 為了防止 Cloudflare bot protection(?) 阻擋,必須加上 Accept-Language。 // TODO: get language from system infomation // dm5.js using "Microsoft-IIS/8.5", // needs 'Accept-Language' to search! 'Accept-Language' : 'zh-TW,zh;q=0.9' + ',ja;q=0.8' + ',en;q=0.7', // DNT : 1, Connection : 'keep-alive', 'Upgrade-Insecure-Requests' : 1, // 'sec-ch-ua-mobile' : '?0', // 'sec-ch-ua-platform' : 'Windows', // TE 請求型頭部用來指定用戶代理希望使用的傳輸編碼類型。 // 可以將其非正式稱為 Accept-Transfer-Encoding,這個名稱顯得更直觀一些。 // 當 TE 設置為 trailers 時,如果服務端支持並且返回了 // Transfer-Encoding: trailers,那麼同時也必須返回另一個響應標頭 Trailer, // TE : 'Trailers', // Origin : URL_options_to_fetch.protocol + '://' + // URL_options_to_fetch.host Pragma : 'no-cache', // 'max-age=0' 'Cache-Control' : 'no-cache' }, options.fetch_type === 'image' ? { // /\.(jpg|png|webp)$/i.test(URL_to_fetch.replace(/[?#].*/, '')) Accept : // 每次請求重設這些標頭。 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8', 'Sec-Fetch-Dest' : /* options.fetch_type || */'image', 'Sec-Fetch-Mode' : 'no-cors', 'Sec-Fetch-Site' : 'cross-site', 'Sec-Fetch-User' : undefined } : { // 為了順暢使用 Cloudflare,必須加上 Sec-Fetch-headers? e.g., mymhh.js // https://blog.kalan.dev/fetch-metadata-request-headers/ 'Sec-Fetch-Dest' : options.fetch_type || 'document', 'Sec-Fetch-Mode' : 'navigate', 'Sec-Fetch-Site' : 'none', 'Sec-Fetch-User' : '?1', }, options.headers, URL_options_to_fetch.headers); for ( var key in URL_options_to_fetch.headers) { if (URL_options_to_fetch.headers[key] === undefined) delete URL_options_to_fetch.headers[key]; } // delete URL_options_to_fetch.headers.Referer; // console.log(options.headers); if (false && /Chrome\//.test(get_URL_node.default_user_agent)) { Object.assign(URL_options_to_fetch.headers, { 'sec-ch-ua' : // '" Not A;Brand";v="99", "Chromium";v="98", "Google Chrome";v="98"' }); } if (node_zlib.gunzipSync // && node_zlib.deflateSync ) { // 早期 node v0.10.25 無 zlib.gunzipSync。Added in: v0.11.12 // 'gzip, deflate, *' URL_options_to_fetch.headers['Accept-Encoding'] = 'gzip, deflate' // Added in: v11.7.0, v10.16.0 + (node_zlib.brotliDecompressSync ? ', br' : ''); } // console.trace(URL_options_to_fetch.headers); if (false) { // @see jQuery if (!options.crossDomain && !URL_options_to_fetch.headers["X-Requested-With"]) { URL_options_to_fetch.headers["X-Requested-With"] = "XMLHttpRequest"; } } if (post_data) { URL_options_to_fetch.method = 'POST'; var _post_data = post_data === FORCE_POST ? '' : post_data; if (false && options.form_data) { // console.log('-'.repeat(79)); // console.log(_post_data.to_Array().content_length); // console.log(_post_data); } Object.assign(URL_options_to_fetch.headers, { 'Content-Type' : options.headers // && options.headers['Content-Type'] // || (options.form_data ? 'multipart/form-data; boundary=' // boundary 存入→ post_data.boundary + _post_data.boundary : 'application/x-www-form-urlencoded'), // prevent HTTP 411 錯誤 – 需要內容長度頭 (411 Length Required) 'Content-Length' : options.form_data // ? _post_data.to_Array().content_length // NG: _post_data.length : charset ? Buffer.byteLength(_post_data, charset) : Buffer .byteLength(_post_data) }); } if (options.method) { // e.g., 'HEAD' URL_options_to_fetch.method = options.method; } URL_options_to_fetch.agent = agent; set_cookie_to_URL_object(URL_options_to_fetch, agent); if (library_namespace.is_debug(3)) { library_namespace.debug('Set headers: ' + JSON.stringify(URL_options_to_fetch.headers), 3, 'get_URL_node'); console.log(URL_options_to_fetch.headers); } // console.log(URL_options_to_fetch); try { // console.trace([ URL_to_fetch, URL_options_to_fetch ]); // request scheme request = URL_is_https ? node_https : node_http; // from node.js 10.9.0 // http.request(url[, options][, callback]) // request: Class: http.ClientRequest request = request.request(URL_options_to_fetch, _onload); } catch (e) { // e.g., _http_client.js:52 if (false) { throw new TypeError( // gettext_config:{"id":"request-path-contains-unescaped-characters"} 'Request path contains unescaped characters'); } _onfail(e); return; } if (options.max_listeners >= 0) { /** * 最多平行取得檔案的數量。 incase "MaxListenersExceededWarning: Possible EventEmitter memory leak detected. 11 connect listeners added. Use emitter.setMaxListeners() to increase limit" */ request.setMaxListeners(options.max_listeners); } if (post_data) { var _post_data = post_data === FORCE_POST ? '' : post_data; // console.log(_post_data); if (options.form_data) { (function write_to_request(data) { if (Array.isArray(data)) { data.forEach(function(chunk) { write_to_request(chunk); }) } else { request.write(data); } })(_post_data.to_Array()); } else if (typeof _post_data === 'string') { library_namespace.debug('set post data: length ' + _post_data.length, 3, 'get_URL_node'); if (_post_data) { library_namespace.debug('set post data: ' + (_post_data.length <= 800 // || library_namespace.is_debug(6) ? _post_data : _post_data.slice(0, 800) + '...'), 3, 'get_URL_node'); } request.write(_post_data); } else { library_namespace.error({ // gettext_config:{"id":"invalid-post-data-$1"} T : [ 'Invalid POST data: %1', JSON.stringify(post_data) ] }); } } /** {Natural}timeout in ms for get URL. 逾時ms數 */ var timeout = options.timeout || get_URL_node.default_timeout, timeout_id, // _ontimeout = function(e) { // 可能已被註銷。 if (finished) { return; } try { // http://hylom.net/node-http-request-get-and-timeout // timeoutイベントは発生しているものの、イベント発生後も引き続きレスポンスを待ち続けている // request.end(); // request.abort(); request.destroy(); } catch (err) { // TODO: handle exception } if (!options.no_warning) { library_namespace.info([ 'get_URL_node: ', { // gettext_config:{"id":"connection-timeout-for-$1-$2"} T : [ 'Connection timeout for %1: [%2]', // time_message(timeout), URL_to_fetch ] } ]); } if (!e) { e = new Error('Timeout ' + time_message(timeout) + ': [' + URL_to_fetch + ']'); e.code = 'TIMEOUT'; } _onfail(e); }; if (timeout > 0) { // setTimeout method 1 // 此方法似乎不能確實於時間到時截斷。或許因為正在 handshaking? request.setTimeout(timeout); // https://nodejs.org/api/http.html#http_request_settimeout_timeout_callback // http://stackoverflow.com/questions/14727115/whats-the-difference-between-req-settimeout-socket-settimeout request.on('timeout', _ontimeout); // setTimeout method 2 // {Object}timeout_id @ node.js timeout_id = setTimeout(_ontimeout, timeout); library_namespace.debug({ // gettext_config:{"id":"add-timeout-$1-$2"} T : [ 'Add timeout %1: [%2]', time_message(timeout), URL_to_fetch ] }, 2, 'get_URL_node'); } else if (timeout) { library_namespace.warn([ 'get_URL_node: ', { // gettext_config:{"id":"invalid-timeout-$1"} T : [ 'Invalid timeout: %1', timeout ] } ]); } library_namespace.debug('set onerror: ' + (options.onfail ? 'user defined' : 'default handler'), 3, 'get_URL_node'); request.on('error', _onfail); // 遇到 "Unhandled 'error' event",或許是 print 到 stdout 時出錯了,不一定是本函數的問題。 // debug error: socket parse error // CloudFlare 遇到 HPE_INVALID_CONSTANT,可能是因為需要 encodeURI(url)。 if (false && library_namespace.is_debug(6)) { request.on('socket', function(socket) { if (socket.parser) { socket.parser._execute = socket.parser.execute; socket.parser.execute = function(d) { console.log(d.toString()); socket.parser._execute(d); }; } // console.log('-----------------------------------'); // console.log(socket.parser.execute); }); } var start_time = (new Date).getTime(); request.end(); } /** * default user agent. for some server, (e.g., tools.wmflabs.org) * Requests must have a user agent. * * @see https://meta.wikimedia.org/wiki/User-Agent_policy * * @type {String} */ get_URL_node.default_user_agent = library_namespace.Class + '/' + library_namespace.version + ' (https://github.com/kanasimi/CeJS)'; // 逾時ms數: 20 minutes get_URL_node.default_timeout = 20 * 60 * 1000; get_URL_node.connects_limit = 100; get_URL_node.get_status = function(item) { var status = { connections : get_URL_node_connections, requests : get_URL_node_requests }; return item ? status[item] : status; }; // setup/reset node agent. function setup_node(type, options) { if (!is_nodejs) return; if (_.get_URL !== get_URL_node) { // 初始化。 node_url = require('url'); node_http = require('http'); node_https = require('https'); node_zlib = require('zlib'); try { node_http2 = require('http2'); } catch (e) { // Is old version. Added in: v8.4.0 } _.get_URL = library_namespace .copy_properties(get_URL, get_URL_node); } if (type !== undefined) { if (typeof type === 'string') type = /^https/i.test(type); var agent = type ? new node_https.Agent(options) : new node_http.Agent(options); if (options && options.as_default) { if (type) { node_https_agent = agent; } else { node_http_agent = agent; } } return agent; } node_http_agent = new node_http.Agent; node_https_agent = new node_https.Agent; // 不需要。 // node_http_agent.maxSockets = 1; // node_https_agent.maxSockets = 1; } _.setup_node_net = setup_node; // CeL.application.net.Ajax.setup_node_net(); // library_namespace.application.net.Ajax.setup_node_net(); setup_node(); // ---------------------------------------------------------------------// function parse_proxy_server(proxy_server) { // console.log(proxy_server); proxy_server = library_namespace.URI(proxy_server); proxy_server.proxy = proxy_server.href; // console.log(proxy_server); return proxy_server; if (typeof proxy_server !== 'string') { return proxy_server; } // href=protocol:(//)?username:password@hostname:port/path/filename?search#hash // 代理伺服器 proxy_server: "username:password@hostname:port" // [ all, protocol, username, password, hostname, port ] var matched = proxy_server .match(/^(?:(https?:)\/\/)?(?:([^:@]+)(?::([^@]*))?@)?([^:@]+)(?::(\d{1,5}))?$/); if (!matched) { return false; } matched = { proxy : matched[0], protocol : matched[1], username : matched[2], password : matched[3], hostname : matched[4], port : +matched[5] }; return matched; } function get_proxy_URL(proxy_server, URL_options_to_fetch, URL_to_fetch) { var proxy_URL = { host : proxy_server.hostname, port : proxy_server.port || library_namespace.net.port_of_protocol[URL_options_to_fetch.protocol .replace(/:$/, '')] || library_namespace.net.port_of_protocol.https, path : URL_to_fetch, protocol : URL_options_to_fetch.protocol, // method: 'GET', headers : { Host : URL_options_to_fetch.host } }; if (proxy_server.agent) { proxy_URL.agent = proxy_server.agent; } if (proxy_server.username) { // https://developer.mozilla.org/zh-TW/docs/Web/HTTP/Authentication // https://developer.mozilla.org/zh-TW/docs/Web/HTTP/Headers/Proxy-Authorization proxy_URL.headers['Proxy-Authorization'] = 'Basic ' + Buffer.from( // proxy.auth proxy_server.username + ':' + (proxy_server.password || '') // ).toString('base64'); } return proxy_URL; } // http://luoxia.me/code/2017/07/16/%E8%81%8A%E8%81%8AAgent&Proxy/ // https://github.com/TooTallNate/node-https-proxy-agent /** * get https:// through proxy 用於取得https網站。 * * @see https://www.vanamco.com/2014/06/24/proxy-requests-in-node-js/ * https://gist.github.com/matthias-christen/6beb3b4dda26bd6a221d * * modify from: * * HTTPS Agent for node.js HTTPS requests via a proxy. * blog.vanamco.com/connecting-via-proxy-node-js/ */ function HttpsProxyAgent(proxy_server, options) { if (!(this instanceof HttpsProxyAgent)) return new HttpsProxyAgent(proxy_server, options); node_https.Agent.call(this, options || {}); this.options = Object.assign({}, options); // href=protocol:(//)?username:password@hostname:port/path/filename?search#hash // 代理伺服器 proxy_server: "username:password@hostname:port" proxy_server = library_namespace.URI(proxy_server); if (!proxy_server) { // gettext_config:{"id":"must-specify-proxy-server-hostname-port"} throw new Error('Must specify proxy server: hostname:port') } this.proxy_server = proxy_server; // https://github.com/nodejs/node/blob/master/lib/net.js // function connect(...args) { } this.createConnection = function connect_proxy_server(options, callback) { // do a CONNECT request var request = Object.assign(get_proxy_URL(proxy_server, options, options.host + ':' + options.port), { method : 'CONNECT' }); // console.log(request); // 此時若有 agent,會是 https。但是待會要用 http connect。 delete request.agent; delete request.protocol; if (proxy_server.agent) set_cookie_to_URL_object(request, proxy_server.agent); library_namespace.debug('Connect to ' + request.path, 2, 'HttpsProxyAgent.createConnection'); request = node_http.request(request); request.on('connect', function(response, socket, headers) { var tls = require('tls'); // https://github.com/nodejs/node/issues/27384 // node.js v12 disable TLS v1.0 and v1.1 by default // tls.DEFAULT_MIN_VERSION = 'TLSv1'; // a tls.TLSSocket object var tls_socket = tls.connect({ host : options.host, socket : socket }, function() { callback(null, tls_socket); }); }); request.on('error', callback); request.end(); } } /** * node_https.request({ // like you'd do it usually... host : 'twitter.com', port : 443, method : 'GET', path : '/', // set proxy agent : new CeL.HttpsProxyAgent('localhost:8080') }, function(resonse) { resonse.on('data', function(data) { console.log(data.toString()); }); }).end(); */ _.HttpsProxyAgent = HttpsProxyAgent; var node_util; if (is_nodejs) { node_util = require('util'); node_util.inherits(HttpsProxyAgent, node_https.Agent); } // https://github.com/nodejs/node/blob/master/lib/_http_agent.js HttpsProxyAgent.prototype.getName = function getName(options) { var name = (options.host || 'localhost') + ':' // + (options.port || '') + ':' + (options.path || ''); return name; }; // Almost verbatim copy of http.Agent.addRequest // https://github.com/nodejs/node/blob/master/lib/_http_agent.js // Agent.prototype.addRequest = function addRequest(req, options HttpsProxyAgent.prototype.addRequest = function addRequest(request, options) { // Get the key for a given set of request options // Agent.prototype.getName var name = this.getName(options); if (!this.sockets[name]) this.sockets[name] = []; // Do not use this.freeSockets if (this.sockets[name].length < this.maxSockets) { options.request = request; // If we are under maxSockets create a new one. this.createSocket(name, options, function(error, tls_socket) { if (error) { process.nextTick(function emitErrorNT(emitter, error) { emitter.emit('error', error); }, request, error); return; } // setRequestSocket(agent, request, socket); request.onSocket(tls_socket); // TODO: set this_agent.options.timeout }); } else { // We are over limit so we'll add it to the queue. if (!this.requests[name]) this.requests[name] = []; this.requests[name].push(request); // lost options??? } }; // Almost verbatim copy of http.Agent.createSocket // https://github.com/nodejs/node/blob/master/lib/_http_agent.js // Agent.prototype.createSocket HttpsProxyAgent.prototype.createSocket = function createSocket(name, options, callback) { var this_agent = this; options = Object.assign({}, options, this.options); options.servername = options.host; if (options.request) { var hostHeader = options.request.getHeader('host'); if (hostHeader) options.servername = hostHeader.replace(/:.*$/, ''); } var called = false; function oncreate(error, tls_socket) { if (called) return; called = true; if (error) { error.message += ' while connecting to HTTP(S) proxy server ' + this_agent.hostname + ':' + this_agent.port; if (options.request) options.request.emit('error', error); else throw error; return; } var name = this_agent.getName(options); if (!this_agent.sockets[name]) this_agent.sockets[name] = []; this_agent.sockets[name].push(tls_socket); // ------------------------ // function installListeners(agent, socket, options) var onFree = function onFree() { this_agent.emit('free', tls_socket, options); }; var onClose = function onClose(error) { /** * This is the only place where sockets get removed from the * Agent. * * If you want to remove a socket from the pool, just close it. * * All socket errors end in a close event anyway. */ this_agent.removeSocket(tls_socket, options); }; var onRemove = function onRemove() { /** * We need this function for cases like HTTP 'upgrade' (defined * by WebSockets) where we need to remove a socket from the pool * because it'll be locked up indefinitely */ this_agent.removeSocket(tls_socket, options); tls_socket.removeListener('close', onClose); tls_socket.removeListener('free', onFree); tls_socket.removeListener('agentRemove', onRemove); }; tls_socket.on('free', onFree); tls_socket.on('close', onClose); tls_socket.on('agentRemove', onRemove); // ------------------------ // assert: error === null callback(error, tls_socket); } // call connect_proxy_server() this_agent.createConnection(options, oncreate); }; // ---------------------------------------------------------------------// // https://github.com/TooTallNate/node-socks-proxy-agent function SocksProxyAgent() { TODO; } // ---------------------------------------------------------------------// // TODO: for non-nodejs /** {Object|Function}fs in node.js */ var node_fs; try { if (is_nodejs) { // @see https://nodejs.org/api/fs.html node_fs = require('fs'); } if (typeof node_fs.readFile !== 'function') { throw true; } } catch (e) { library_namespace.debug([ this.id + ': ', { // gettext_config:{"id":"there-is-no-`fs`-package-for-node.js-so-there-is-no-file-operation-function-for-node"} T : '無 node.js 之 `fs` 套件,因此不具備 node 之檔案操作功能。' } ]); if (false) { // enumerate for get_URL_cache_node // 模擬 node.js 之 fs,以達成最起碼的效果(即無 cache 功能的情況)。 node_fs = { readFile : function(file_name, charset, callback) { callback(true); }, writeFile : library_namespace.null_function }; } } /** * cache 相關函數: @see application.storage.file.get_cache_file application.OS.Windows.file.cacher application.net.Ajax.get_URL_cache application.net.wiki wiki_API.cache() CeL.wiki.cache() */ /** * cache 作業操作之輔助套裝函數。 * * 注意: 若執行 onload() 時沒提供 XMLHttp,表示採用 cache。 * * TODO: 以 HEAD 檢測,若有新版本則不採用 cache。 * * @param {String|Object}URL * 欲請求之目的 URL or options * @param {Function}[onload] * callback when successful loaded. onload(data, error, XMLHttp) * @param {Object}[options] * 附加參數/設定特殊功能與選項 */ function get_URL_cache_node(URL, onload, options) { if (typeof options === 'string') { // auto-detecting options = /\.[a-z\d\-]+$/i.test(options) ? { file_name : options } : /[\\\/]+$/i.test(options) // || 也可以測試是不是目錄、此目錄是否存在。 ? { directory : options } : { encoding : options }; } else if (!library_namespace.is_Object(options)) { // 前置處理。 options = Object.create(null); } var file_name = options.file_name, /** {String}file encoding for fs of node.js. */ encoding = 'encoding' in options ? options.encoding : get_URL_cache_node.encoding; if (!file_name && (file_name = decodeURI(URL).match(/[^\/]+$/))) { file_name = file_name[0]; if (library_namespace.HTML_to_Unicode) { // 去掉 "&" 之類。 file_name = library_namespace.HTML_to_Unicode(file_name); } file_name = library_namespace.to_file_name(file_name); library_namespace.debug([ { // gettext_config:{"id":"get-filename-from-url-$1"} T : [ '自 URL 取得檔名:%1', URL ] }, '\n→ ' + file_name ], 1, 'get_URL_cache_node'); } if (typeof options.file_name_processor === 'function') { file_name = options.file_name_processor(file_name); } if (!file_name) { // gettext_config:{"id":"no-file-name-specified"} onload(undefined, new Error('No file name specified.')); return; } if (options.directory) { file_name = library_namespace.append_path_separator( options.directory, file_name); } var file_status; try { file_status = node_fs.statSync(file_name); } catch (e) { // TODO: handle exception } if (!options.get_contents && options.web_resource_date && file_status) { // download newer only if ((file_status.mtimeMs || file_status.mtime) // - Date.parse(options.web_resource_date) > -1) { library_namespace.debug('File on web (' + options.web_resource_date + ') is not newer than local file (' + file_status.mtime + '): ' + file_name + '', 1, 'get_URL_cache_node'); onload(undefined, _.get_URL_cache.NO_NEWS); return; } } library_namespace.debug([ { // gettext_config:{"id":"download-$1"} T : [ '下載 %1', URL ] }, '\n→ ', { // gettext_config:{"id":"$1-(file-encoding-$2-charset-$3)"} T : [ '%1 (file encoding %2, charset %3)', file_name, encoding, options.charset ] } ], 1, 'get_URL_cache_node'); node_fs.readFile(file_name, encoding, // function(error, data) { // console.trace([ error, !!data, options.reget ]); // 警告: 對於從 HTTP 標頭獲得文件名的情況,就算不設定 options.reget 也沒用,還是會重新獲取檔案。 // options.force_download if (!options.reget) { if (!error && options.web_resource_date && file_status) { // download newer only if ((file_status.mtimeMs || file_status.mtime) // - Date.parse(options.web_resource_date) > -1) { // No new file on web. onload(data); return; } } if (!error && !options.preserve_newer // 若是容許空內容,應該特別指定 options.allow_empty。 && (data || options.allow_empty)) { library_namespace.debug({ // gettext_config:{"id":"using-cached-data"} T : 'Using cached data.' }, 3, 'get_URL_cache_node'); library_namespace.debug('Cached data: [' + data.slice(0, 200) + ']...', 5, 'get_URL_cache_node'); // TODO: use cached_status onload(data); return; } library_namespace.debug({ // gettext_config:{"id":"no-valid-cached-data.-try-to-get-data-(again)"} T : 'No valid cached data. Try to get data (again)...' }, 3, 'get_URL_cache_node'); } _.get_URL(URL, function(XMLHttp, error) { if (error) { library_namespace.error([ 'get_URL_cache_node.cache: ', { // gettext_config:{"id":"got-error-when-retrieving-$1-$2"} T : [ 'Got error when retrieving [%1]: %2', // URL, localize_error(error) ] } ]); // WARNING: XMLHttp 僅在重新取得 URL 時提供。 onload(undefined, error, XMLHttp); return; } // .buffer: node only. data = !encoding && XMLHttp.buffer || XMLHttp.response || XMLHttp.responseText; // 資料事後處理程序 (post-processor): // 將以 .postprocessor() 的回傳作為要處理的資料。 if (typeof options.postprocessor === 'function') { data = options.postprocessor(data, XMLHttp); } if (options.file_name_processor) { file_name = options.file_name_processor(file_name, // header_filename XMLHttp.filename); } else if (XMLHttp.filename) { if (false) { console.log([ options.directory, options.file_name, XMLHttp.filename ]); console.log(XMLHttp.headers); } if (!options.file_name) { file_name = (options.directory || '') // 若是沒有特別設置檔名,則改採用header裡面的檔名。 + XMLHttp.filename; library_namespace.info([ 'get_URL_cache_node: ', { // gettext_config:{"id":"got-file-name-from-http-header-$1"} T : [ 'Got file name from HTTP header: %1', XMLHttp.filename ] } ]); } else if (!options.file_name.endsWith(XMLHttp.filename)) { library_namespace.info([ 'get_URL_cache_node: ', { T : [ // gettext_config:{"id":"set-file-name-$1-file-name-from-header-$2"} 'Set file name: [%1], file name from header: [%2].' // , options.file_name, XMLHttp.filename ] } ]); } } var URL_date = XMLHttp.headers['date']; if (URL_date && options.preserve_newer && file_status) { // data.length === stat.size if ((file_status.mtimeMs || file_status.mtime) // - Date.parse(URL_date) > -1) { // Local file is newer than file on web. onload(data.toString(), undefined, XMLHttp); return; } } /** * 寫入cache。 * * 若出現錯誤,則不寫入cache。 */ if (data && /[^\\\/]$/.test(file_name)) { XMLHttp.cached_file_path = file_name; if (!file_status) { try { file_status = node_fs.statSync(file_name); if (file_status && !options.reget) { library_namespace.info([ 'get_URL_cache_node.cache: ', { // gettext_config:{"id":"find-that-the-file-exists-after-get-the-file-from-web-$1-.-do-not-overwrite-it"} T : [ '重新獲取檔案後發現原檔案已存在,跳過不覆寫:[%1]', file_name ] } ]); onload(data.toString(), undefined, XMLHttp); return; } } catch (e) { // TODO: handle exception } } if (!options.no_write_info) { library_namespace.info([ 'get_URL_cache_node.cache: ', { // gettext_config:{"id":"write-data-to-cache-file-$1"} T : [ 'Write data to cache file [%1].', file_name ] } ]); library_namespace.debug({ // gettext_config:{"id":"the-data-to-cache-$1"} T : [ 'The data to cache: %1...', // data && JSON.stringify(data).slice(0, 190) ] }, 3, 'get_URL_cache_node'); } try { node_fs.writeFileSync(file_name, data, encoding); } catch (error) { onload(data.toString(), error, XMLHttp); return; } } // set file modify date // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Date // https://tools.ietf.org/html/rfc7231#section-7.1.1.2 if (URL_date) { try { // The "Date" header field represents the date // and time at which the message was originated // fs.utimesSync(path, atime, mtime) // atime: the last time this file was accessed node_fs.utimesSync(file_name, new Date, // mtime: the last time this file was modified URL_date); } catch (e) { // TODO: handle exception } } // Warning: 已經有些程式碼預設會回傳 {String} onload(data.toString(), undefined, XMLHttp); }, // character encoding of HTML web page // is different from the file we want to save to options.charset, options.post_data, options.get_URL_options); }); } /** {String}預設 file encoding for fs of node.js。 */ get_URL_cache_node.encoding = 'utf8'; if (is_nodejs) { _.get_URL_cache = get_URL_cache_node; _.get_URL_cache.NO_NEWS = typeof Symbol === 'function' ? Symbol('no news') : { 'no news' : true }; } // ---------------------------------------------------------------------// /** * defective polyfill for W3C fetch API * * @since 2018/10/16 17:47:12 * @deprecated */ function node_fetch(input, init) { // TODO: input is a Request object. var url = input instanceof URL ? input : new URL(input.toString()), options = library_namespace .new_options(init); function executor(resolve, reject) { function callback(response) { if ((response.statusCode / 100 | 0) === 3 && response.headers.location && response.headers.location !== url.toString()) { try { // request.abort(); request.destroy(); } catch (e) { } Object.assign(options, { redirected : true, initial_URL : options.initial_URL || input }); url = new URL(response.headers.location, url); library_namespace.debug({ // gettext_config:{"id":"$1-redirecting-to-$2-←-$3"} T : [ '%1 Redirecting to [%2] ← [%3]', response.statusCode, url.toString(), input ] }, 1, 'fetch'); node_fetch(url, options); return; } /** {Array} [ {Buffer}, {Buffer}, ... ] */ var data = [], length = 0; response.on('data', function(chunk) { // {Buffer}chunk length += chunk.length; library_namespace.debug('receive BODY ' // + chunk.length + '/' + length + ': ' + url, 4, 'fetch'); data.push(chunk); }); response.on('end', function() { library_namespace.debug('end(): ' + url, 2, 'fetch'); // console.log('No more data in response: ' + url); // it is faster to provide the length explicitly. data = Buffer.concat(data, length); var result_Object = { // https://developer.mozilla.org/zh-TW/docs/Web/API/Response // https://nodejs.org/api/http.html#http_http_get_options_callback url : options.initial_URL || input, headers : response.headers, status : response.statusCode, statusText : response.statusMessage, ok : (response.statusCode / 100 | 0) === 2, redirected : !!options.redirected, useFinalURL : url.toString(), _buffer : data, // TODO: body : new ReadableStream() // methods of // https://developer.mozilla.org/en-US/docs/Web/API/Body // https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream text : function text() { try { return Promise.resolve( // this._buffer.toString()); } catch (e) { return Promise.reject(e); } }, json : function json() { return this.text().then(JSON.parse); }, arrayBuffer : function arrayBuffer() { return Promise.resolve(this._buffer.buffer); } }; resolve(result_Object); }); } // TODO: add normal headers // CloudFlare 必須設定好headers才能才會才允許回傳資料。 if (library_namespace.is_debug(9) && library_namespace.env.has_console) { console.trace([ url.toString(), options ]); } // https://nodejs.org/api/http.html var request = url.protocol === 'http:' ? node_http.request(url .toString(), options, callback) : node_https.request(url .toString(), options, callback); request.on('error', reject); if (options.body) request.write(options.body); request.end(); } return new Promise(executor); } /** * defective polyfill for W3C fetch API * * 必須額外設定 credentials。 * * TODO: fetch 預設上不傳送或接收任何 cookies,如果網站依賴 session 會導致請求回傳未經認證,需要使用 cookies * * @examples var fetch = CeL.fetch; fetch(url).then(function(response) { return response.json(); }).then(function(json) { console.log(json); }); fetch(url).then(function(response) { return response.text(); }).then(function(html) { console.log(html); }); * * @see 20181016.import_earthquake_shakemap.js * @see https://fetch.spec.whatwg.org/#fetch-method * https://developer.mozilla.org/zh-TW/docs/Web/API/Fetch_API * https://github.com/node-fetch/node-fetch * * @since 2021/8/4 6:6:45 */ function fetch__get_URL(input, init) { function executor(resolve, reject) { function callback(XMLHttp, error) { if (error) { reject(error); return; } // console.trace(XMLHttp); var result_Object = { // https://developer.mozilla.org/zh-TW/docs/Web/API/Response // https://nodejs.org/api/http.html#http_http_get_options_callback headers : XMLHttp.headers, ok : (XMLHttp.status / 100 | 0) === 2, redirected : XMLHttp.redirected, status : XMLHttp.status, statusText : XMLHttp.statusText, // 重定向後獲得的最終 URL。 url : XMLHttp.responseURL, useFinalURL : true, _buffer : XMLHttp.buffer, // TODO: body : new ReadableStream() // methods of // https://developer.mozilla.org/en-US/docs/Web/API/Body // https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream text : function text() { try { return Promise.resolve( // this._buffer.toString()); } catch (e) { return Promise.reject(e); } }, json : function json() { return this.text().then(JSON.parse); }, arrayBuffer : function arrayBuffer() { return Promise.resolve(this._buffer.buffer); } }; resolve(result_Object); } // CloudFlare 必須設定好 headers 才能才會才允許回傳資料。 // get_URL() 可自動設定 headers。 _.get_URL(input, callback, null, init && init.body, Object.assign({ // headers : { 'User-Agent' : '' }, onfail : reject, agent : true }, init)); } return new Promise(executor); } if (is_nodejs) { _.fetch = fetch__get_URL; } // ---------------------------------------------------------------------// // export 導出. return (_// JSDT:_module_ ); }