added static cors-anywhere

cors-anywhere/lib/cors-anywhere.js

@@ -0,0 +1,431 @@
+// © 2013 - 2016 Rob Wu <rob@robwu.nl>
+// Released under the MIT license
+
+'use strict';
+
+var httpProxy = require('http-proxy');
+var net = require('net');
+var url = require('url');
+var regexp_tld = require('./regexp-top-level-domain');
+var getProxyForUrl = require('proxy-from-env').getProxyForUrl;
+
+var help_text = {};
+function showUsage(help_file, headers, response) {
+  var isHtml = /\.html$/.test(help_file);
+  headers['content-type'] = isHtml ? 'text/html' : 'text/plain';
+  if (help_text[help_file] != null) {
+    response.writeHead(200, headers);
+    response.end(help_text[help_file]);
+  } else {
+    require('fs').readFile(help_file, 'utf8', function(err, data) {
+      if (err) {
+        console.error(err);
+        response.writeHead(500, headers);
+        response.end();
+      } else {
+        help_text[help_file] = data;
+        showUsage(help_file, headers, response); // Recursive call, but since data is a string, the recursion will end
+      }
+    });
+  }
+}
+
+/**
+ * Check whether the specified hostname is valid.
+ *
+ * @param hostname {string} Host name (excluding port) of requested resource.
+ * @return {boolean} Whether the requested resource can be accessed.
+ */
+function isValidHostName(hostname) {
+  return !!(
+    regexp_tld.test(hostname) ||
+    net.isIPv4(hostname) ||
+    net.isIPv6(hostname)
+  );
+}
+
+/**
+ * Adds CORS headers to the response headers.
+ *
+ * @param headers {object} Response headers
+ * @param request {ServerRequest}
+ */
+function withCORS(headers, request) {
+  headers['access-control-allow-origin'] = '*';
+  var corsMaxAge = request.corsAnywhereRequestState.corsMaxAge;
+  if (corsMaxAge) {
+    headers['access-control-max-age'] = corsMaxAge;
+  }
+  if (request.headers['access-control-request-method']) {
+    headers['access-control-allow-methods'] = request.headers['access-control-request-method'];
+    delete request.headers['access-control-request-method'];
+  }
+  if (request.headers['access-control-request-headers']) {
+    headers['access-control-allow-headers'] = request.headers['access-control-request-headers'];
+    delete request.headers['access-control-request-headers'];
+  }
+
+  headers['access-control-expose-headers'] = Object.keys(headers).join(',');
+
+  return headers;
+}
+
+/**
+ * Performs the actual proxy request.
+ *
+ * @param req {ServerRequest} Incoming http request
+ * @param res {ServerResponse} Outgoing (proxied) http request
+ * @param proxy {HttpProxy}
+ */
+function proxyRequest(req, res, proxy) {
+  var location = req.corsAnywhereRequestState.location;
+  req.url = location.path;
+
+  var proxyOptions = {
+    changeOrigin: false,
+    prependPath: false,
+    target: location,
+    headers: {
+      host: location.host,
+    },
+    // HACK: Get hold of the proxyReq object, because we need it later.
+    // https://github.com/nodejitsu/node-http-proxy/blob/v1.11.1/lib/http-proxy/passes/web-incoming.js#L144
+    buffer: {
+      pipe: function(proxyReq) {
+        var proxyReqOn = proxyReq.on;
+        // Intercepts the handler that connects proxyRes to res.
+        // https://github.com/nodejitsu/node-http-proxy/blob/v1.11.1/lib/http-proxy/passes/web-incoming.js#L146-L158
+        proxyReq.on = function(eventName, listener) {
+          if (eventName !== 'response') {
+            return proxyReqOn.call(this, eventName, listener);
+          }
+          return proxyReqOn.call(this, 'response', function(proxyRes) {
+            if (onProxyResponse(proxy, proxyReq, proxyRes, req, res)) {
+              try {
+                listener(proxyRes);
+              } catch (err) {
+                // Wrap in try-catch because an error could occur:
+                // "RangeError: Invalid status code: 0"
+                // https://github.com/Rob--W/cors-anywhere/issues/95
+                // https://github.com/nodejitsu/node-http-proxy/issues/1080
+
+                // Forward error (will ultimately emit the 'error' event on our proxy object):
+                // https://github.com/nodejitsu/node-http-proxy/blob/v1.11.1/lib/http-proxy/passes/web-incoming.js#L134
+                proxyReq.emit('error', err);
+              }
+            }
+          });
+        };
+        return req.pipe(proxyReq);
+      },
+    },
+  };
+
+  var proxyThroughUrl = req.corsAnywhereRequestState.getProxyForUrl(location.href);
+  if (proxyThroughUrl) {
+    proxyOptions.target = proxyThroughUrl;
+    proxyOptions.toProxy = true;
+    // If a proxy URL was set, req.url must be an absolute URL. Then the request will not be sent
+    // directly to the proxied URL, but through another proxy.
+    req.url = location.href;
+  }
+
+  // Start proxying the request
+  try {
+    proxy.web(req, res, proxyOptions);
+  } catch (err) {
+    proxy.emit('error', err, req, res);
+  }
+}
+
+/**
+ * This method modifies the response headers of the proxied response.
+ * If a redirect is detected, the response is not sent to the client,
+ * and a new request is initiated.
+ *
+ * client (req) -> CORS Anywhere -> (proxyReq) -> other server
+ * client (res) <- CORS Anywhere <- (proxyRes) <- other server
+ *
+ * @param proxy {HttpProxy}
+ * @param proxyReq {ClientRequest} The outgoing request to the other server.
+ * @param proxyRes {ServerResponse} The response from the other server.
+ * @param req {IncomingMessage} Incoming HTTP request, augmented with property corsAnywhereRequestState
+ * @param req.corsAnywhereRequestState {object}
+ * @param req.corsAnywhereRequestState.location {object} See parseURL
+ * @param req.corsAnywhereRequestState.getProxyForUrl {function} See proxyRequest
+ * @param req.corsAnywhereRequestState.proxyBaseUrl {string} Base URL of the CORS API endpoint
+ * @param req.corsAnywhereRequestState.maxRedirects {number} Maximum number of redirects
+ * @param req.corsAnywhereRequestState.redirectCount_ {number} Internally used to count redirects
+ * @param res {ServerResponse} Outgoing response to the client that wanted to proxy the HTTP request.
+ *
+ * @returns {boolean} true if http-proxy should continue to pipe proxyRes to res.
+ */
+function onProxyResponse(proxy, proxyReq, proxyRes, req, res) {
+  var requestState = req.corsAnywhereRequestState;
+
+  var statusCode = proxyRes.statusCode;
+
+  if (!requestState.redirectCount_) {
+    res.setHeader('x-request-url', requestState.location.href);
+  }
+  // Handle redirects
+  if (statusCode === 301 || statusCode === 302 || statusCode === 303 || statusCode === 307 || statusCode === 308) {
+    var locationHeader = proxyRes.headers.location;
+    if (locationHeader) {
+      locationHeader = url.resolve(requestState.location.href, locationHeader);
+
+      if (statusCode === 301 || statusCode === 302 || statusCode === 303) {
+        // Exclude 307 & 308, because they are rare, and require preserving the method + request body
+        requestState.redirectCount_ = requestState.redirectCount_ + 1 || 1;
+        if (requestState.redirectCount_ <= requestState.maxRedirects) {
+          // Handle redirects within the server, because some clients (e.g. Android Stock Browser)
+          // cancel redirects.
+          // Set header for debugging purposes. Do not try to parse it!
+          res.setHeader('X-CORS-Redirect-' + requestState.redirectCount_, statusCode + ' ' + locationHeader);
+
+          req.method = 'GET';
+          req.headers['content-length'] = '0';
+          delete req.headers['content-type'];
+          requestState.location = parseURL(locationHeader);
+
+          // Remove all listeners (=reset events to initial state)
+          req.removeAllListeners();
+
+          // Remove the error listener so that the ECONNRESET "error" that
+          // may occur after aborting a request does not propagate to res.
+          // https://github.com/nodejitsu/node-http-proxy/blob/v1.11.1/lib/http-proxy/passes/web-incoming.js#L134
+          proxyReq.removeAllListeners('error');
+          proxyReq.once('error', function catchAndIgnoreError() {});
+          proxyReq.abort();
+
+          // Initiate a new proxy request.
+          proxyRequest(req, res, proxy);
+          return false;
+        }
+      }
+      proxyRes.headers.location = requestState.proxyBaseUrl + '/' + locationHeader;
+    }
+  }
+
+  // Strip cookies
+  delete proxyRes.headers['set-cookie'];
+  delete proxyRes.headers['set-cookie2'];
+
+  proxyRes.headers['x-final-url'] = requestState.location.href;
+  withCORS(proxyRes.headers, req);
+  return true;
+}
+
+
+/**
+ * @param req_url {string} The requested URL (scheme is optional).
+ * @return {object} URL parsed using url.parse
+ */
+function parseURL(req_url) {
+  var match = req_url.match(/^(?:(https?:)?\/\/)?(([^\/?]+?)(?::(\d{0,5})(?=[\/?]|$))?)([\/?][\S\s]*|$)/i);
+  //                              ^^^^^^^          ^^^^^^^^      ^^^^^^^                ^^^^^^^^^^^^
+  //                            1:protocol       3:hostname     4:port                 5:path + query string
+  //                                              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  //                                            2:host
+  if (!match) {
+    return null;
+  }
+  if (!match[1]) {
+    // Scheme is omitted.
+    if (req_url.lastIndexOf('//', 0) === -1) {
+      // "//" is omitted.
+      req_url = '//' + req_url;
+    }
+    req_url = (match[4] === '443' ? 'https:' : 'http:') + req_url;
+  }
+  return url.parse(req_url);
+}
+
+// Request handler factory
+function getHandler(options, proxy) {
+  var corsAnywhere = {
+    getProxyForUrl: getProxyForUrl, // Function that specifies the proxy to use
+    maxRedirects: 5,                // Maximum number of redirects to be followed.
+    originBlacklist: [],            // Requests from these origins will be blocked.
+    originWhitelist: [],            // If non-empty, requests not from an origin in this list will be blocked.
+    checkRateLimit: null,           // Function that may enforce a rate-limit by returning a non-empty string.
+    redirectSameOrigin: false,      // Redirect the client to the requested URL for same-origin requests.
+    requireHeader: null,            // Require a header to be set?
+    removeHeaders: [],              // Strip these request headers.
+    setHeaders: {},                 // Set these request headers.
+    corsMaxAge: 0,                  // If set, an Access-Control-Max-Age header with this value (in seconds) will be added.
+    helpFile: __dirname + '/help.txt',
+  };
+
+  Object.keys(corsAnywhere).forEach(function(option) {
+    if (Object.prototype.hasOwnProperty.call(options, option)) {
+      corsAnywhere[option] = options[option];
+    }
+  });
+
+  // Convert corsAnywhere.requireHeader to an array of lowercase header names, or null.
+  if (corsAnywhere.requireHeader) {
+    if (typeof corsAnywhere.requireHeader === 'string') {
+      corsAnywhere.requireHeader = [corsAnywhere.requireHeader.toLowerCase()];
+    } else if (!Array.isArray(corsAnywhere.requireHeader) || corsAnywhere.requireHeader.length === 0) {
+      corsAnywhere.requireHeader = null;
+    } else {
+      corsAnywhere.requireHeader = corsAnywhere.requireHeader.map(function(headerName) {
+        return headerName.toLowerCase();
+      });
+    }
+  }
+  var hasRequiredHeaders = function(headers) {
+    return !corsAnywhere.requireHeader || corsAnywhere.requireHeader.some(function(headerName) {
+      return Object.hasOwnProperty.call(headers, headerName);
+    });
+  };
+
+  return function(req, res) {
+    req.corsAnywhereRequestState = {
+      getProxyForUrl: corsAnywhere.getProxyForUrl,
+      maxRedirects: corsAnywhere.maxRedirects,
+      corsMaxAge: corsAnywhere.corsMaxAge,
+    };
+
+    var cors_headers = withCORS({}, req);
+    if (req.method === 'OPTIONS') {
+      // Pre-flight request. Reply successfully:
+      res.writeHead(200, cors_headers);
+      res.end();
+      return;
+    }
+
+    var location = parseURL(req.url.slice(1));
+
+    if (!location) {
+      // Invalid API call. Show how to correctly use the API
+      showUsage(corsAnywhere.helpFile, cors_headers, res);
+      return;
+    }
+
+    if (location.host === 'iscorsneeded') {
+      // Is CORS needed? This path is provided so that API consumers can test whether it's necessary
+      // to use CORS. The server's reply is always No, because if they can read it, then CORS headers
+      // are not necessary.
+      res.writeHead(200, {'Content-Type': 'text/plain'});
+      res.end('no');
+      return;
+    }
+
+    if (location.port > 65535) {
+      // Port is higher than 65535
+      res.writeHead(400, 'Invalid port', cors_headers);
+      res.end('Port number too large: ' + location.port);
+      return;
+    }
+
+    if (!/^\/https?:/.test(req.url) && !isValidHostName(location.hostname)) {
+      // Don't even try to proxy invalid hosts (such as /favicon.ico, /robots.txt)
+      res.writeHead(404, 'Invalid host', cors_headers);
+      res.end('Invalid host: ' + location.hostname);
+      return;
+    }
+
+    if (!hasRequiredHeaders(req.headers)) {
+      res.writeHead(400, 'Header required', cors_headers);
+      res.end('Missing required request header. Must specify one of: ' + corsAnywhere.requireHeader);
+      return;
+    }
+
+    var origin = req.headers.origin || '';
+    if (corsAnywhere.originBlacklist.indexOf(origin) >= 0) {
+      res.writeHead(403, 'Forbidden', cors_headers);
+      res.end('The origin "' + origin + '" was blacklisted by the operator of this proxy.');
+      return;
+    }
+
+    if (corsAnywhere.originWhitelist.length && corsAnywhere.originWhitelist.indexOf(origin) === -1) {
+      res.writeHead(403, 'Forbidden', cors_headers);
+      res.end('The origin "' + origin + '" was not whitelisted by the operator of this proxy.');
+      return;
+    }
+
+    var rateLimitMessage = corsAnywhere.checkRateLimit && corsAnywhere.checkRateLimit(origin);
+    if (rateLimitMessage) {
+      res.writeHead(429, 'Too Many Requests', cors_headers);
+      res.end('The origin "' + origin + '" has sent too many requests.\n' + rateLimitMessage);
+      return;
+    }
+
+    if (corsAnywhere.redirectSameOrigin && origin && location.href[origin.length] === '/' &&
+        location.href.lastIndexOf(origin, 0) === 0) {
+      // Send a permanent redirect to offload the server. Badly coded clients should not waste our resources.
+      cors_headers.vary = 'origin';
+      cors_headers['cache-control'] = 'private';
+      cors_headers.location = location.href;
+      res.writeHead(301, 'Please use a direct request', cors_headers);
+      res.end();
+      return;
+    }
+
+    var isRequestedOverHttps = req.connection.encrypted || /^\s*https/.test(req.headers['x-forwarded-proto']);
+    var proxyBaseUrl = (isRequestedOverHttps ? 'https://' : 'http://') + req.headers.host;
+
+    corsAnywhere.removeHeaders.forEach(function(header) {
+      delete req.headers[header];
+    });
+
+    Object.keys(corsAnywhere.setHeaders).forEach(function(header) {
+      req.headers[header] = corsAnywhere.setHeaders[header];
+    });
+
+    req.corsAnywhereRequestState.location = location;
+    req.corsAnywhereRequestState.proxyBaseUrl = proxyBaseUrl;
+
+    proxyRequest(req, res, proxy);
+  };
+}
+
+// Create server with default and given values
+// Creator still needs to call .listen()
+exports.createServer = function createServer(options) {
+  options = options || {};
+
+  // Default options:
+  var httpProxyOptions = {
+    xfwd: true,            // Append X-Forwarded-* headers
+  };
+  // Allow user to override defaults and add own options
+  if (options.httpProxyOptions) {
+    Object.keys(options.httpProxyOptions).forEach(function(option) {
+      httpProxyOptions[option] = options.httpProxyOptions[option];
+    });
+  }
+
+  var proxy = httpProxy.createServer(httpProxyOptions);
+  var requestHandler = getHandler(options, proxy);
+  var server;
+  if (options.httpsOptions) {
+    server = require('https').createServer(options.httpsOptions, requestHandler);
+  } else {
+    server = require('http').createServer(requestHandler);
+  }
+
+  // When the server fails, just show a 404 instead of Internal server error
+  proxy.on('error', function(err, req, res) {
+    if (res.headersSent) {
+      // This could happen when a protocol error occurs when an error occurs
+      // after the headers have been received (and forwarded). Do not write
+      // the headers because it would generate an error.
+      return;
+    }
+
+    // When the error occurs after setting headers but before writing the response,
+    // then any previously set headers must be removed.
+    var headerNames = res.getHeaderNames ? res.getHeaderNames() : Object.keys(res._headers || {});
+    headerNames.forEach(function(name) {
+      res.removeHeader(name);
+    });
+
+    res.writeHead(404, {'Access-Control-Allow-Origin': '*'});
+    res.end('Not found because of proxy error: ' + err);
+  });
+
+  return server;
+};

cors-anywhere/lib/help.txt

@@ -0,0 +1,30 @@
+This API enables cross-origin requests to anywhere.
+
+Usage:
+
+/               Shows help
+/iscorsneeded   This is the only resource on this host which is served without CORS headers.
+/<url>          Create a request to <url>, and includes CORS headers in the response.
+
+If the protocol is omitted, it defaults to http (https if port 443 is specified).
+
+Cookies are disabled and stripped from requests.
+
+Redirects are automatically followed. For debugging purposes, each followed redirect results
+in the addition of a X-CORS-Redirect-n header, where n starts at 1. These headers are not
+accessible by the XMLHttpRequest API.
+After 5 redirects, redirects are not followed any more. The redirect response is sent back
+to the browser, which can choose to follow the redirect (handled automatically by the browser).
+
+The requested URL is available in the X-Request-URL response header.
+The final URL, after following all redirects, is available in the X-Final-URL response header.
+
+
+To prevent the use of the proxy for casual browsing, the API requires either the Origin
+or the X-Requested-With header to be set. To avoid unnecessary preflight (OPTIONS) requests,
+it's recommended to not manually set these headers in your code.
+
+
+Demo          :   https://robwu.nl/cors-anywhere.html
+Source code   :   https://github.com/Rob--W/cors-anywhere/
+Documentation :   https://github.com/Rob--W/cors-anywhere/#documentation

cors-anywhere/lib/rate-limit.js

@@ -0,0 +1,74 @@
+'use strict';
+module.exports = function createRateLimitChecker(CORSANYWHERE_RATELIMIT) {
+  // Configure rate limit. The following format is accepted for CORSANYWHERE_RATELIMIT:
+  // <max requests per period> <period in minutes> <non-ratelimited hosts>
+  // where <non-ratelimited hosts> is a space-separated list of strings or regexes (/.../) that
+  // matches the whole host (ports have to be listed explicitly if applicable).
+  // <period in minutes> cannot be zero.
+  //
+  // Examples:
+  // - Allow any origin to make one request per 5 minutes:
+  //   1 5
+  //
+  // - Allow example.com to make an unlimited number of requests, and the others 1 per 5 minutes.
+  //   1 5 example.com
+  //
+  // - Allow example.com, or any subdomain to make any number of requests and block the rest:
+  //   0 1 /(.*\.)?example\.com/
+  //
+  // - Allow example.com and www.example.com, and block the rest:
+  //   0 1 example.com www.example.com
+  var rateLimitConfig = /^(\d+) (\d+)(?:\s*$|\s+(.+)$)/.exec(CORSANYWHERE_RATELIMIT);
+  if (!rateLimitConfig) {
+    // No rate limit by default.
+    return function checkRateLimit() {};
+  }
+  var maxRequestsPerPeriod = parseInt(rateLimitConfig[1]);
+  var periodInMinutes = parseInt(rateLimitConfig[2]);
+  var unlimitedPattern = rateLimitConfig[3]; // Will become a RegExp or void.
+  if (unlimitedPattern) {
+    var unlimitedPatternParts = [];
+    unlimitedPattern.trim().split(/\s+/).forEach(function(unlimitedHost, i) {
+      var startsWithSlash = unlimitedHost.charAt(0) === '/';
+      var endsWithSlash = unlimitedHost.slice(-1) === '/';
+      if (startsWithSlash || endsWithSlash) {
+        if (unlimitedHost.length === 1 || !startsWithSlash || !endsWithSlash) {
+          throw new Error('Invalid CORSANYWHERE_RATELIMIT. Regex at index ' + i +
+              ' must start and end with a slash ("/").');
+        }
+        unlimitedHost = unlimitedHost.slice(1, -1);
+        // Throws if the pattern is invalid.
+        new RegExp(unlimitedHost);
+      } else {
+        // Just escape RegExp characters even though they cannot appear in a host name.
+        // The only actual important escape is the dot.
+        unlimitedHost = unlimitedHost.replace(/[$()*+.?[\\\]^{|}]/g, '\\$&');
+      }
+      unlimitedPatternParts.push(unlimitedHost);
+    });
+    unlimitedPattern = new RegExp('^(?:' + unlimitedPatternParts.join('|') + ')$', 'i');
+  }
+
+  var accessedHosts = Object.create(null);
+  setInterval(function() {
+    accessedHosts = Object.create(null);
+  }, periodInMinutes * 60000);
+
+  var rateLimitMessage = 'The number of requests is limited to ' + maxRequestsPerPeriod +
+    (periodInMinutes === 1 ? ' per minute' : ' per ' + periodInMinutes + ' minutes') + '. ' +
+    'Please self-host CORS Anywhere if you need more quota. ' +
+    'See https://github.com/Rob--W/cors-anywhere#demo-server';
+
+  return function checkRateLimit(origin) {
+    var host = origin.replace(/^[\w\-]+:\/\//i, '');
+    if (unlimitedPattern && unlimitedPattern.test(host)) {
+      return;
+    }
+    var count = accessedHosts[host] || 0;
+    ++count;
+    if (count > maxRequestsPerPeriod) {
+      return rateLimitMessage;
+    }
+    accessedHosts[host] = count;
+  };
+};