covid19/generate.js

#!/usr/bin/env node

const path = require('path');
const fs = require('fs');
const parseCsv = require('csv-parse/lib/sync');
const pug = require('pug');

const publicDir = path.join(__dirname, 'public');
const templatesDir = path.join(__dirname, 'tmpl');
const dataDir = path.resolve(path.join(__dirname, '..', 'COVID-19', 'csse_covid_19_data'));
const timeSeriesDir = path.join(dataDir, 'csse_covid_19_time_series');

const promiseMe = (fn) => {
	return new Promise((resolve, reject) => {
		fn((err, result) => {
			if (err) {
				reject(err);
				return;
			}

			resolve(result);
		});
	});
};

fs.mkdirSync(path.join(__dirname, 'public', 'countries'), {
	recursive: true,
});

fs.copyFileSync(
	path.join(__dirname, 'node_modules', 'chart.js', 'dist', 'Chart.bundle.min.js'),
	path.join(publicDir, 'Chart.bundle.js'),
);

fs.copyFileSync(
	path.join(__dirname, 'node_modules', 'bootstrap', 'dist', 'css', 'bootstrap.css'),
	path.join(publicDir, 'bootstrap.css'),
);

const deathsGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_global.csv');
const confirmedGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_global.csv');
const deathsUSCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_US.csv');
const confirmedUSCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_US.csv');

const zeroPad = value => value < 10 ? `0${value}` : value.toString();

const toSafeName = x => x.replace(/[^A-Za-z]/g, '-').toLowerCase();

const processGlobalDeaths = async () => {
	const globalStart = Date.now();
	let start = Date.now();

	console.log('reading CSV...');
	const timeSeriesGlobalRaw = fs.readFileSync(deathsGlobalCsv, {encoding: 'utf8'});
	console.log(`read CSV in ${Date.now() - start}ms`);

	start = Date.now();
	let tsGlobalRecords = parseCsv(timeSeriesGlobalRaw, {
		cast: true,
		columns: true,
	});
	console.log(`parsed CSV in ${Date.now() - start}ms`);
	//
	// tsGlobalRecords = tsGlobalRecords.filter((record) => {
	// 	return record['Country/Region'] === 'Australia';
	// });

	start = Date.now();
	tsGlobalRecords.forEach((record) => {
		record.timeSeriesDaily = [];
		record.timeSeriesMonthly = [];
		const dateColumns = Object.keys(record).filter(x => /^\d+\/\d+\/\d+$/.test(x))
			.map(key => {
				return {
					key,
					date: new Date(key),
				};
			})
			.sort((a, b) => {
				if (a.date.getTime() === b.date.getTime()) {
					return 0;
				}

				return a.date.getTime() < b.date.getTime() ? -1 : 1;
			});

		const toSortableDate = date => [
			date.getUTCFullYear(),
			zeroPad(date.getUTCMonth() + 1),
			zeroPad(date.getUTCDate()),
		].join('-');

		dateColumns.forEach((obj) => {
			const value = Number(record[obj.key]) || 0;
			const date = obj.date;
			delete record[obj.key];
			const sortableKey = toSortableDate(date);
			const lastItem = record.timeSeriesDaily[record.timeSeriesDaily.length - 1];
			record.timeSeriesDaily.push({
				key: sortableKey,
				value,
				delta: lastItem ? value - lastItem.value : 0,
			});
		});

		const monthlyTotals = [];
		const monthlyMaxes = {};
		record.timeSeriesDaily.forEach((item) => {
			const key = item.key.replace(/-\d+$/, '');
			if (!(key in monthlyMaxes)) {
				monthlyMaxes[key] = {
					date: item.key,
					value: item.value,
				};
			} else if (item.key > monthlyMaxes[key].date) {
				monthlyMaxes[key] = {
					date: item.key,
					value: item.value,
				}
			}
		});

		Object.keys(monthlyMaxes).forEach((key) => {
			monthlyTotals.push({
				key,
				value: monthlyMaxes[key].value,
			});
		});

		monthlyTotals.sort((a, b) => a.key.localeCompare(b.key));
		monthlyTotals.forEach((item, i) => {
			const prev = monthlyTotals[i - 1];
			item.delta = prev ? item.value - prev.value : 0;
		});

		record.total = record.timeSeriesDaily.length ?
			record.timeSeriesDaily[record.timeSeriesDaily.length - 1].value :
			0;
		record.timeSeriesMonthly = monthlyTotals;

		record.state = record['Province/State'];
		record.country = record['Country/Region'];
		record.lat = record.Lat;
		record.long = record.Long;
		record.countrySafeName = toSafeName(record.country);
		record.stateSafeName = toSafeName(record.state);

		delete record['Province/State'];
		delete record['Country/Region'];
		delete record.Lat;
		delete record.Long;
	});

	tsGlobalRecords.sort((a, b) => {
		if (a.country === b.country) {
			return a.state.localeCompare(b.state);
		}

		return a.country.localeCompare(b.country);
	});

	const perCountryTotals = {};
	tsGlobalRecords.forEach((record) => {
		perCountryTotals[record.country] = perCountryTotals[record.country] || {
			total: 0,
			timeSeriesDaily: {},
			timeSeriesMonthly: {},
			states: [],
			safeName: record.countrySafeName,
		};

		const item = perCountryTotals[record.country];
		item.total += record.total;
		item.states.push(record);

		record.timeSeriesDaily.forEach((ts) => {
			item.timeSeriesDaily[ts.key] = item.timeSeriesDaily[ts.key] || {
				value: 0,
				delta: 0,
			};
			item.timeSeriesDaily[ts.key].value += ts.value;
			item.timeSeriesDaily[ts.key].delta += ts.delta;
		});

		record.timeSeriesMonthly.forEach((ts) => {
			item.timeSeriesMonthly[ts.key] = item.timeSeriesMonthly[ts.key] || {
				value: 0,
				delta: 0,
			};
			item.timeSeriesMonthly[ts.key].value += ts.value;
			item.timeSeriesMonthly[ts.key].delta += ts.delta;
		});
	});

	const countryArr = Object.keys(perCountryTotals).map((countryName) => {
		const item = perCountryTotals[countryName];
		return {
			name: countryName,
			safeName: item.safeName,
			total: item.total,
			states: item.states,
			timeSeriesDaily: Object.keys(item.timeSeriesDaily).sort().map((date) => {
				return {
					key: date,
					value: item.timeSeriesDaily[date].value,
					delta: item.timeSeriesDaily[date].delta,
				};
			}),
			timeSeriesMonthly: Object.keys(item.timeSeriesMonthly).sort().map((date) => {
				return {
					key: date,
					value: item.timeSeriesMonthly[date].value,
					delta: item.timeSeriesMonthly[date].delta,
				};
			}),
		};
	});

	console.log(`transformed data in ${Date.now() - start}ms`);

	start = Date.now();
	const allTmpl = path.join(templatesDir, 'all.pug');
	const allHtml = pug.renderFile(allTmpl, {
		data: countryArr,
	});

	const targetFile = path.join(publicDir, 'index.html');
	fs.writeFileSync(targetFile, allHtml);
	console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);
	// fs.writeFileSync(path.join(publicDir, 'countries.json'), JSON.stringify(countryArr, null, '  '));

	const singleCountryTmpl = path.join(templatesDir, 'country.pug');
	await Promise.all(countryArr.map(async (countryData) => {
		const start = Date.now();
		const targetFile = path.join(publicDir, 'countries', countryData.safeName + '.html');
		const countryHtml = pug.renderFile(singleCountryTmpl, {
			data: countryData,
		});
		console.log(`writing to ${targetFile}`);
		await promiseMe(callback => fs.writeFile(targetFile, countryHtml, callback));
		console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);
	}));

	console.log(`finished in ${((Date.now() - globalStart) / 1000).toFixed(2)}s`);
};

processGlobalDeaths()
	.then(() => {
		console.log('all done');
	})
	.catch((err) => {
		console.log(err);
		process.exit(1);
	});