From 3836f7c40a58a1a11a6c5bdbe36b63fe9fa51c59 Mon Sep 17 00:00:00 2001
From: Jamie Curnow <jc@jc21.com>
Date: Thu, 3 Jan 2019 17:04:53 +1000
Subject: [PATCH] Fetch ip ranges for CDN servers, Cloudfront and Cloudfare
---
rootfs/etc/nginx/conf.d/default.conf | 3 +
.../etc/nginx/conf.d/include/ip_ranges.conf | 174 ++++++++++++++++++
src/backend/index.js | 3 +
src/backend/internal/ip_ranges.js | 163 ++++++++++++++++
src/backend/logger.js | 17 +-
src/backend/templates/ip_ranges.conf | 3 +
6 files changed, 355 insertions(+), 8 deletions(-)
create mode 100644 rootfs/etc/nginx/conf.d/include/ip_ranges.conf
create mode 100644 src/backend/internal/ip_ranges.js
create mode 100644 src/backend/templates/ip_ranges.conf
diff --git a/rootfs/etc/nginx/conf.d/default.conf b/rootfs/etc/nginx/conf.d/default.conf
index 490e286..729b94a 100644
--- a/rootfs/etc/nginx/conf.d/default.conf
+++ b/rootfs/etc/nginx/conf.d/default.conf
@@ -1,3 +1,6 @@
+# Generated IP Ranges for safe real IP determination
+include conf.d/include/ip_ranges.conf;
+
# Healthcheck Host which proxies to the Manager,
# thus the healthcheck ensures both services are running
server {
diff --git a/rootfs/etc/nginx/conf.d/include/ip_ranges.conf b/rootfs/etc/nginx/conf.d/include/ip_ranges.conf
new file mode 100644
index 0000000..0289cec
--- /dev/null
+++ b/rootfs/etc/nginx/conf.d/include/ip_ranges.conf
@@ -0,0 +1,174 @@
+
+set_real_ip_from 13.124.199.0/24;
+
+set_real_ip_from 34.226.14.0/24;
+
+set_real_ip_from 52.124.128.0/17;
+
+set_real_ip_from 54.230.0.0/16;
+
+set_real_ip_from 54.239.128.0/18;
+
+set_real_ip_from 52.82.128.0/19;
+
+set_real_ip_from 99.84.0.0/16;
+
+set_real_ip_from 52.15.127.128/26;
+
+set_real_ip_from 35.158.136.0/24;
+
+set_real_ip_from 52.57.254.0/24;
+
+set_real_ip_from 18.216.170.128/25;
+
+set_real_ip_from 13.54.63.128/26;
+
+set_real_ip_from 13.59.250.0/26;
+
+set_real_ip_from 13.210.67.128/26;
+
+set_real_ip_from 35.167.191.128/26;
+
+set_real_ip_from 52.47.139.0/24;
+
+set_real_ip_from 52.199.127.192/26;
+
+set_real_ip_from 52.212.248.0/26;
+
+set_real_ip_from 205.251.192.0/19;
+
+set_real_ip_from 52.66.194.128/26;
+
+set_real_ip_from 54.239.192.0/19;
+
+set_real_ip_from 70.132.0.0/18;
+
+set_real_ip_from 13.32.0.0/15;
+
+set_real_ip_from 13.113.203.0/24;
+
+set_real_ip_from 34.195.252.0/24;
+
+set_real_ip_from 35.162.63.192/26;
+
+set_real_ip_from 34.223.12.224/27;
+
+set_real_ip_from 13.35.0.0/16;
+
+set_real_ip_from 204.246.172.0/23;
+
+set_real_ip_from 204.246.164.0/22;
+
+set_real_ip_from 52.56.127.0/25;
+
+set_real_ip_from 204.246.168.0/22;
+
+set_real_ip_from 13.228.69.0/24;
+
+set_real_ip_from 34.216.51.0/25;
+
+set_real_ip_from 71.152.0.0/17;
+
+set_real_ip_from 216.137.32.0/19;
+
+set_real_ip_from 205.251.249.0/24;
+
+set_real_ip_from 99.86.0.0/16;
+
+set_real_ip_from 52.46.0.0/18;
+
+set_real_ip_from 52.84.0.0/15;
+
+set_real_ip_from 54.233.255.128/26;
+
+set_real_ip_from 64.252.64.0/18;
+
+set_real_ip_from 52.52.191.128/26;
+
+set_real_ip_from 204.246.174.0/23;
+
+set_real_ip_from 64.252.128.0/18;
+
+set_real_ip_from 205.251.254.0/24;
+
+set_real_ip_from 143.204.0.0/16;
+
+set_real_ip_from 205.251.252.0/23;
+
+set_real_ip_from 52.78.247.128/26;
+
+set_real_ip_from 204.246.176.0/20;
+
+set_real_ip_from 52.220.191.0/26;
+
+set_real_ip_from 13.249.0.0/16;
+
+set_real_ip_from 54.240.128.0/18;
+
+set_real_ip_from 205.251.250.0/23;
+
+set_real_ip_from 52.222.128.0/17;
+
+set_real_ip_from 54.182.0.0/16;
+
+set_real_ip_from 54.192.0.0/16;
+
+set_real_ip_from 34.232.163.208/29;
+
+set_real_ip_from 2600:9000:eee::/48;
+
+set_real_ip_from 2600:9000:4000::/36;
+
+set_real_ip_from 2600:9000:3000::/36;
+
+set_real_ip_from 2600:9000:f000::/36;
+
+set_real_ip_from 2600:9000:fff::/48;
+
+set_real_ip_from 2600:9000:2000::/36;
+
+set_real_ip_from 2600:9000:1000::/36;
+
+set_real_ip_from 2600:9000:ddd::/48;
+
+set_real_ip_from 2600:9000:5300::/40;
+
+set_real_ip_from 103.21.244.0/22;
+
+set_real_ip_from 103.22.200.0/22;
+
+set_real_ip_from 103.31.4.0/22;
+
+set_real_ip_from 104.16.0.0/12;
+
+set_real_ip_from 108.162.192.0/18;
+
+set_real_ip_from 131.0.72.0/22;
+
+set_real_ip_from 141.101.64.0/18;
+
+set_real_ip_from 162.158.0.0/15;
+
+set_real_ip_from 172.64.0.0/13;
+
+set_real_ip_from 173.245.48.0/20;
+
+set_real_ip_from 188.114.96.0/20;
+
+set_real_ip_from 190.93.240.0/20;
+
+set_real_ip_from 197.234.240.0/22;
+
+set_real_ip_from 198.41.128.0/17;
+
+set_real_ip_from 2400:cb00::/32;
+
+set_real_ip_from 2405:b500::/32;
+
+set_real_ip_from 2606:4700::/32;
+
+set_real_ip_from 2803:f800::/32;
+
+set_real_ip_from 2c0f:f248::/32;
+
+set_real_ip_from 2a06:98c0::/29;
diff --git a/src/backend/index.js b/src/backend/index.js
index d646df0..cd0a781 100644
--- a/src/backend/index.js
+++ b/src/backend/index.js
@@ -11,6 +11,7 @@ function appStart () {
const app = require('./app');
const apiValidator = require('./lib/validator/api');
const internalCertificate = require('./internal/certificate');
+ const internalIpRanges = require('./internal/ip_ranges');
return migrate.latest()
.then(setup)
@@ -18,9 +19,11 @@ function appStart () {
.then(() => {
return apiValidator.loadSchemas;
})
+ .then(internalIpRanges.fetch)
.then(() => {
internalCertificate.initTimer();
+ internalIpRanges.initTimer();
const server = app.listen(81, () => {
logger.info('PID ' + process.pid + ' listening on port 81 ...');
diff --git a/src/backend/internal/ip_ranges.js b/src/backend/internal/ip_ranges.js
new file mode 100644
index 0000000..41c221f
--- /dev/null
+++ b/src/backend/internal/ip_ranges.js
@@ -0,0 +1,163 @@
+'use strict';
+
+const https = require('https');
+const fs = require('fs');
+const _ = require('lodash');
+const logger = require('../logger').ip_ranges;
+const debug_mode = process.env.NODE_ENV !== 'production';
+const error = require('../lib/error');
+const internalNginx = require('./nginx');
+const Liquid = require('liquidjs');
+
+const CLOUDFRONT_URL = 'https://ip-ranges.amazonaws.com/ip-ranges.json';
+const CLOUDFARE_V4_URL = 'https://www.cloudflare.com/ips-v4';
+const CLOUDFARE_V6_URL = 'https://www.cloudflare.com/ips-v6';
+
+const internalIpRanges = {
+
+ interval_timeout: 1000 * 60 * 60 * 6, // 6 hours
+ interval: null,
+ interval_processing: false,
+ iteration_count: 0,
+
+ initTimer: () => {
+ logger.info('IP Ranges Renewal Timer initialized');
+ internalIpRanges.interval = setInterval(internalIpRanges.fetch, internalIpRanges.interval_timeout);
+ },
+
+ fetchUrl: url => {
+ return new Promise((resolve, reject) => {
+ logger.info('Fetching ' + url);
+ return https.get(url, res => {
+ res.setEncoding('utf8');
+ let raw_data = '';
+ res.on('data', chunk => {
+ raw_data += chunk;
+ });
+
+ res.on('end', () => {
+ resolve(raw_data);
+ });
+ }).on('error', err => {
+ reject(err);
+ });
+ });
+ },
+
+ /**
+ * Triggered at startup and then later by a timer, this will fetch the ip ranges from services and apply them to nginx.
+ */
+ fetch: () => {
+ if (!internalIpRanges.interval_processing) {
+ internalIpRanges.interval_processing = true;
+ logger.info('Fetching IP Ranges from online services...');
+
+ let ip_ranges = [];
+
+ return internalIpRanges.fetchUrl(CLOUDFRONT_URL)
+ .then(cloudfront_data => {
+ let data = JSON.parse(cloudfront_data);
+
+ if (data && typeof data.prefixes !== 'undefined') {
+ data.prefixes.map(item => {
+ if (item.service === 'CLOUDFRONT') {
+ ip_ranges.push(item.ip_prefix);
+ }
+ });
+ }
+
+ if (data && typeof data.ipv6_prefixes !== 'undefined') {
+ data.ipv6_prefixes.map(item => {
+ if (item.service === 'CLOUDFRONT') {
+ ip_ranges.push(item.ipv6_prefix);
+ }
+ });
+ }
+ })
+ .then(() => {
+ return internalIpRanges.fetchUrl(CLOUDFARE_V4_URL);
+ })
+ .then(cloudfare_data => {
+ let items = cloudfare_data.split('\n');
+ ip_ranges = [... ip_ranges, ... items];
+ })
+ .then(() => {
+ return internalIpRanges.fetchUrl(CLOUDFARE_V6_URL);
+ })
+ .then(cloudfare_data => {
+ let items = cloudfare_data.split('\n');
+ ip_ranges = [... ip_ranges, ... items];
+ })
+ .then(() => {
+ let clean_ip_ranges = [];
+ ip_ranges.map(range => {
+ if (range) {
+ clean_ip_ranges.push(range);
+ }
+ });
+
+ return internalIpRanges.generateConfig(clean_ip_ranges)
+ .then(() => {
+ if (internalIpRanges.iteration_count) {
+ // Reload nginx
+ return internalNginx.reload();
+ }
+ });
+ })
+ .then(() => {
+ internalIpRanges.interval_processing = false;
+ internalIpRanges.iteration_count++;
+ })
+ .catch(err => {
+ logger.error(err.message);
+ internalIpRanges.interval_processing = false;
+ });
+ }
+ },
+
+ /**
+ * @param {Array} ip_ranges
+ * @returns {Promise}
+ */
+ generateConfig: (ip_ranges) => {
+ if (debug_mode) {
+ logger.info('Generating IP Ranges Config', ip_ranges);
+ }
+
+ let renderEngine = Liquid({
+ root: __dirname + '/../templates/'
+ });
+
+ return new Promise((resolve, reject) => {
+ let template = null;
+ let filename = '/etc/nginx/conf.d/include/ip_ranges.conf';
+ try {
+ template = fs.readFileSync(__dirname + '/../templates/ip_ranges.conf', {encoding: 'utf8'});
+ } catch (err) {
+ reject(new error.ConfigurationError(err.message));
+ return;
+ }
+
+ renderEngine
+ .parseAndRender(template, {ip_ranges: ip_ranges})
+ .then(config_text => {
+ fs.writeFileSync(filename, config_text, {encoding: 'utf8'});
+
+ if (debug_mode) {
+ logger.debug('Wrote config:', filename, config_text);
+ }
+
+ resolve(true);
+ })
+ .catch(err => {
+ if (debug_mode) {
+ logger.warn('Could not write ' + filename + ':', err.message);
+ }
+
+ reject(new error.ConfigurationError(err.message));
+ });
+ });
+ }
+};
+
+module.exports = internalIpRanges;
diff --git a/src/backend/logger.js b/src/backend/logger.js
index 589b327..b8f00f7 100644
--- a/src/backend/logger.js
+++ b/src/backend/logger.js
@@ -1,12 +1,13 @@
const {Signale} = require('signale');
module.exports = {
- global: new Signale({scope: 'Global '}),
- migrate: new Signale({scope: 'Migrate '}),
- express: new Signale({scope: 'Express '}),
- access: new Signale({scope: 'Access '}),
- nginx: new Signale({scope: 'Nginx '}),
- ssl: new Signale({scope: 'SSL '}),
- import: new Signale({scope: 'Importer'}),
- setup: new Signale({scope: 'Setup '})
+ global: new Signale({scope: 'Global '}),
+ migrate: new Signale({scope: 'Migrate '}),
+ express: new Signale({scope: 'Express '}),
+ access: new Signale({scope: 'Access '}),
+ nginx: new Signale({scope: 'Nginx '}),
+ ssl: new Signale({scope: 'SSL '}),
+ import: new Signale({scope: 'Importer '}),
+ setup: new Signale({scope: 'Setup '}),
+ ip_ranges: new Signale({scope: 'IP Ranges'})
};
diff --git a/src/backend/templates/ip_ranges.conf b/src/backend/templates/ip_ranges.conf
new file mode 100644
index 0000000..8ede2bd
--- /dev/null
+++ b/src/backend/templates/ip_ranges.conf
@@ -0,0 +1,3 @@
+{% for range in ip_ranges %}
+set_real_ip_from {{ range }};
+{% endfor %}
\ No newline at end of file