covid19/generate.js

#!/usr/bin/env node

const path = require('path');
const fs = require('fs');
const parseCsv = require('csv-parse/lib/sync');
const pug = require('pug');

const publicDir = path.join(__dirname, 'public');
const templatesDir = path.join(__dirname, 'tmpl');
const dataDir = path.resolve(path.join(__dirname, '..', 'COVID-19', 'csse_covid_19_data'));
const timeSeriesDir = path.join(dataDir, 'csse_covid_19_time_series');

const promiseMe = (fn) => {
	return new Promise((resolve, reject) => {
		fn((err, result) => {
			if (err) {
				reject(err);
				return;
			}

			resolve(result);
		});
	});
};

fs.mkdirSync(path.join(__dirname, 'public', 'countries'), {
	recursive: true,
});

fs.copyFileSync(
	path.join(__dirname, 'node_modules', 'chart.js', 'dist', 'Chart.bundle.min.js'),
	path.join(publicDir, 'Chart.bundle.js'),
);

fs.copyFileSync(
	path.join(__dirname, 'node_modules', 'bootstrap', 'dist', 'css', 'bootstrap.css'),
	path.join(publicDir, 'bootstrap.css'),
);

const deathsGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_global.csv');
const confirmedGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_global.csv');
const deathsUSCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_US.csv');
const confirmedUSCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_US.csv');

const zeroPad = value => value < 10 ? `0${value}` : value.toString();
const toSafeName = x => x.replace(/[^A-Za-z]/g, '-').toLowerCase();

const processGlobalDeaths = async () => {
	const globalStart = Date.now();
	let start = Date.now();

	const timeSeriesGlobalRaw = fs.readFileSync(deathsGlobalCsv, {encoding: 'utf8'});
	console.log(`read global deaths CSV in ${Date.now() - start}ms`);

	start = Date.now();
	const timeSeriesUSRaw = fs.readFileSync(deathsUSCsv, { encoding: 'utf8' });
	console.log(`read US deaths CSV in ${Date.now() - start}ms`);

	start = Date.now();
	let tsGlobalRecords = parseCsv(timeSeriesGlobalRaw, {
		cast: true,
		columns: true,
	});
	console.log(`parsed global deaths CSV in ${Date.now() - start}ms`);

	start = Date.now();
	let tsUSRecords = parseCsv(timeSeriesUSRaw, {
		cast: true,
		columns: true,
	});
	console.log(`parsed US deaths CSV in ${Date.now() - start}ms`);

	// tsGlobalRecords = tsGlobalRecords.filter((record) => {
	// 	return record['Country/Region'] === 'US';
	// });

	// state/county data is separated for the US and doesn't need to be rolled up
	tsUSRecords.forEach((usRecord) => {
		const newRecord = {
			...usRecord,
			needsRollup: false,
			Long: usRecord.Long_,
			'Province/State': usRecord.Province_State,
			'Country/Region': usRecord.Country_Region,
		};

		delete newRecord.UID;
		delete newRecord.iso2;
		delete newRecord.iso3;
		delete newRecord.code3;
		delete newRecord.FIPS;
		delete newRecord.Combined_Key;
		delete newRecord.Long_;
		delete newRecord.Province_State;
		delete newRecord.Country_Region;

		tsGlobalRecords.push(newRecord);
	});

	start = Date.now();
	tsGlobalRecords.forEach((record) => {
		record.timeSeriesDaily = [];
		record.timeSeriesMonthly = [];
		const dateColumns = Object.keys(record).filter(x => /^\d+\/\d+\/\d+$/.test(x))
			.map(key => {
				return {
					key,
					date: new Date(key),
				};
			})
			.sort((a, b) => {
				if (a.date.getTime() === b.date.getTime()) {
					return 0;
				}

				return a.date.getTime() < b.date.getTime() ? -1 : 1;
			});

		const toSortableDate = date => [
			date.getUTCFullYear(),
			zeroPad(date.getUTCMonth() + 1),
			zeroPad(date.getUTCDate()),
		].join('-');

		dateColumns.forEach((obj) => {
			const value = Number(record[obj.key]) || 0;
			const date = obj.date;
			delete record[obj.key];
			const sortableKey = toSortableDate(date);
			const lastItem = record.timeSeriesDaily[record.timeSeriesDaily.length - 1];
			record.timeSeriesDaily.push({
				key: sortableKey,
				value,
				delta: lastItem ? value - lastItem.value : 0,
			});
		});

		const monthlyTotals = [];
		const monthlyMaxes = {};
		record.timeSeriesDaily.forEach((item) => {
			const key = item.key.replace(/-\d+$/, '');
			if (!(key in monthlyMaxes)) {
				monthlyMaxes[key] = {
					date: item.key,
					value: item.value,
				};
			} else if (item.key > monthlyMaxes[key].date) {
				monthlyMaxes[key] = {
					date: item.key,
					value: item.value,
				}
			}
		});

		Object.keys(monthlyMaxes).forEach((key) => {
			monthlyTotals.push({
				key,
				value: monthlyMaxes[key].value,
			});
		});

		monthlyTotals.sort((a, b) => a.key.localeCompare(b.key));
		monthlyTotals.forEach((item, i) => {
			const prev = monthlyTotals[i - 1];
			item.delta = prev ? item.value - prev.value : 0;
		});

		record.total = record.timeSeriesDaily.length ?
			record.timeSeriesDaily[record.timeSeriesDaily.length - 1].value :
			0;
		record.timeSeriesMonthly = monthlyTotals;

		record.state = record['Province/State'];
		record.country = record['Country/Region'];
		record.lat = record.Lat;
		record.long = record.Long;
		record.county = record.Admin2 || '';
		record.population = record.Population || null;

		if (record.country === 'US') {
			record.country = 'United States';
		}

		record.countrySafeName = toSafeName(record.country);
		record.stateSafeName = toSafeName(record.state);
		record.countySafeName = toSafeName(record.county);

		delete record['Province/State'];
		delete record['Country/Region'];
		delete record.Lat;
		delete record.Long;
		delete record.Admin2;
		delete record.Population;
	});

	tsGlobalRecords.sort((a, b) => {
		if (a.country === b.country) {
			return a.state.localeCompare(b.state);
		}

		return a.country.localeCompare(b.country);
	});

	const perCountryTotals = {};
	const perStateTotals = {};
	tsGlobalRecords.forEach((record) => {
		perCountryTotals[record.country] = perCountryTotals[record.country] || {
			total: 0,
			timeSeriesDaily: {},
			timeSeriesMonthly: {},
			states: [],
			safeName: record.countrySafeName,
		};

		const item = perCountryTotals[record.country];

		// roll up up state/county data
		if (record.state && record.county) {
			const stateItem = perStateTotals[record.state] = perStateTotals[record.state] || {
				name: record.state,
				safeName: record.stateSafeName,
				country: record.country,
				countrySafeName: record.countrySafeName,
				total: 0,
				timeSeriesDaily: {},
				timeSeriesMonthly: {},
				counties: [],
			};

			stateItem.total += record.total;
			record.timeSeriesDaily.forEach((ts) => {
				stateItem.timeSeriesDaily[ts.key] = stateItem.timeSeriesDaily[ts.key] || {
					value: 0,
					delta: 0,
				};
				stateItem.timeSeriesDaily[ts.key].value += ts.value;
				stateItem.timeSeriesDaily[ts.key].delta += ts.delta;
			});

			record.timeSeriesMonthly.forEach((ts) => {
				stateItem.timeSeriesMonthly[ts.key] = stateItem.timeSeriesMonthly[ts.key] || {
					value: 0,
					delta: 0,
				};
				stateItem.timeSeriesMonthly[ts.key].value += ts.value;
				stateItem.timeSeriesMonthly[ts.key].delta += ts.delta;
			});

			stateItem.counties.push(record);
		} else {
			item.states.push(record);
		}

		if (record.needsRollup === false) {
			return;
		}

		item.total += record.total;
		record.timeSeriesDaily.forEach((ts) => {
			item.timeSeriesDaily[ts.key] = item.timeSeriesDaily[ts.key] || {
				value: 0,
				delta: 0,
			};
			item.timeSeriesDaily[ts.key].value += ts.value;
			item.timeSeriesDaily[ts.key].delta += ts.delta;
		});

		record.timeSeriesMonthly.forEach((ts) => {
			item.timeSeriesMonthly[ts.key] = item.timeSeriesMonthly[ts.key] || {
				value: 0,
				delta: 0,
			};
			item.timeSeriesMonthly[ts.key].value += ts.value;
			item.timeSeriesMonthly[ts.key].delta += ts.delta;
		});
	});

	Object.keys(perStateTotals).forEach((stateName) => {
		const item = perStateTotals[stateName];
		const stateItem = {
			name: stateName,
			safeName: item.safeName,
			country: item.country,
			countrySafeName: item.countrySafeName,
			total: item.total,
			counties: item.counties,
			timeSeriesDaily: Object.keys(item.timeSeriesDaily).sort().map((date) => {
				return {
					key: date,
					value: item.timeSeriesDaily[date].value,
					delta: item.timeSeriesDaily[date].delta,
				};
			}),
			timeSeriesMonthly: Object.keys(item.timeSeriesMonthly).sort().map((date) => {
				return {
					key: date,
					value: item.timeSeriesMonthly[date].value,
					delta: item.timeSeriesMonthly[date].delta,
				};
			}),
		};

		// insert into states array for the country
		perCountryTotals[item.country].states.push(stateItem);
	});

	const countryArr = Object.keys(perCountryTotals).map((countryName) => {
		const item = perCountryTotals[countryName];
		return {
			name: countryName,
			safeName: item.safeName,
			total: item.total,
			states: item.states,
			timeSeriesDaily: Object.keys(item.timeSeriesDaily).sort().map((date) => {
				return {
					key: date,
					value: item.timeSeriesDaily[date].value,
					delta: item.timeSeriesDaily[date].delta,
				};
			}),
			timeSeriesMonthly: Object.keys(item.timeSeriesMonthly).sort().map((date) => {
				return {
					key: date,
					value: item.timeSeriesMonthly[date].value,
					delta: item.timeSeriesMonthly[date].delta,
				};
			}),
		};
	});

	console.log(`transformed data in ${Date.now() - start}ms`);

	start = Date.now();
	const allTmpl = path.join(templatesDir, 'all.pug');
	const allHtml = pug.renderFile(allTmpl, {
		data: countryArr,
		$title: 'The World'
	});

	const targetFile = path.join(publicDir, 'index.html');
	fs.writeFileSync(targetFile, allHtml);
	console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);
	// fs.writeFileSync(path.join(publicDir, 'countries.json'), JSON.stringify(countryArr, null, '  '));

	const singleCountryTmpl = path.join(templatesDir, 'country.pug');
	const singleStateTmpl = path.join(templatesDir, 'state.pug');
	await Promise.all(countryArr.map(async (countryData) => {
		const start = Date.now();
		const targetFile = path.join(publicDir, 'countries', countryData.safeName + '.html');
		const countryHtml = pug.renderFile(singleCountryTmpl, {
			data: countryData,
			$title: countryData.name,
		});
		console.log(`writing to ${targetFile}`);
		await promiseMe(callback => fs.writeFile(targetFile, countryHtml, callback));
		console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);

		if (countryData.states.length) {
			for (const stateData of countryData.states) {
				if (!stateData.name || !stateData.counties || !stateData.counties.length) {
					continue;
				}
				const start = Date.now();
				const targetFile = path.join(publicDir, 'countries', countryData.safeName + '-' + stateData.safeName + '.html');
				const stateHtml = pug.renderFile(singleStateTmpl, {
					data: stateData,
					$title: stateData.name + ' - ' + countryData.name,
				});
				await promiseMe(callback => fs.writeFile(targetFile, stateHtml, callback));
				console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);
			}
		}
	}));

	console.log(`finished in ${((Date.now() - globalStart) / 1000).toFixed(2)}s`);
};

processGlobalDeaths()
	.then(() => {
		console.log('all done');
	})
	.catch((err) => {
		console.log(err);
		process.exit(1);
	});