#!/usr/bin/env node const path = require('path'); const fs = require('fs'); const parseCsv = require('csv-parse/lib/sync'); const pug = require('pug'); const publicDir = path.join(__dirname, 'public'); const templatesDir = path.join(__dirname, 'tmpl'); const dataDir = path.resolve(path.join(__dirname, '..', 'COVID-19', 'csse_covid_19_data')); const timeSeriesDir = path.join(dataDir, 'csse_covid_19_time_series'); const promiseMe = (fn) => { return new Promise((resolve, reject) => { fn((err, result) => { if (err) { reject(err); return; } resolve(result); }); }); }; fs.mkdirSync(path.join(__dirname, 'public', 'countries'), { recursive: true, }); fs.copyFileSync( path.join(__dirname, 'node_modules', 'chart.js', 'dist', 'Chart.bundle.min.js'), path.join(publicDir, 'Chart.bundle.js'), ); fs.copyFileSync( path.join(__dirname, 'node_modules', 'bootstrap', 'dist', 'css', 'bootstrap.css'), path.join(publicDir, 'bootstrap.css'), ); const deathsGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_global.csv'); const confirmedGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_global.csv'); const deathsUSCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_US.csv'); const confirmedUSCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_US.csv'); const zeroPad = value => value < 10 ? `0${value}` : value.toString(); const toSafeName = x => x.replace(/[^A-Za-z]/g, '-').toLowerCase(); const processGlobalDeaths = async () => { const globalStart = Date.now(); let start = Date.now(); console.log('reading CSV...'); const timeSeriesGlobalRaw = fs.readFileSync(deathsGlobalCsv, {encoding: 'utf8'}); console.log(`read CSV in ${Date.now() - start}ms`); start = Date.now(); let tsGlobalRecords = parseCsv(timeSeriesGlobalRaw, { cast: true, columns: true, }); console.log(`parsed CSV in ${Date.now() - start}ms`); // // tsGlobalRecords = tsGlobalRecords.filter((record) => { // return record['Country/Region'] === 'Australia'; // }); start = Date.now(); tsGlobalRecords.forEach((record) => { record.timeSeriesDaily = []; record.timeSeriesMonthly = []; const dateColumns = Object.keys(record).filter(x => /^\d+\/\d+\/\d+$/.test(x)) .map(key => { return { key, date: new Date(key), }; }) .sort((a, b) => { if (a.date.getTime() === b.date.getTime()) { return 0; } return a.date.getTime() < b.date.getTime() ? -1 : 1; }); const toSortableDate = date => [ date.getUTCFullYear(), zeroPad(date.getUTCMonth() + 1), zeroPad(date.getUTCDate()), ].join('-'); dateColumns.forEach((obj) => { const value = Number(record[obj.key]) || 0; const date = obj.date; delete record[obj.key]; const sortableKey = toSortableDate(date); const lastItem = record.timeSeriesDaily[record.timeSeriesDaily.length - 1]; record.timeSeriesDaily.push({ key: sortableKey, value, delta: lastItem ? value - lastItem.value : 0, }); }); const monthlyTotals = []; const monthlyMaxes = {}; record.timeSeriesDaily.forEach((item) => { const key = item.key.replace(/-\d+$/, ''); if (!(key in monthlyMaxes)) { monthlyMaxes[key] = { date: item.key, value: item.value, }; } else if (item.key > monthlyMaxes[key].date) { monthlyMaxes[key] = { date: item.key, value: item.value, } } }); Object.keys(monthlyMaxes).forEach((key) => { monthlyTotals.push({ key, value: monthlyMaxes[key].value, }); }); monthlyTotals.sort((a, b) => a.key.localeCompare(b.key)); monthlyTotals.forEach((item, i) => { const prev = monthlyTotals[i - 1]; item.delta = prev ? item.value - prev.value : 0; }); record.total = record.timeSeriesDaily.length ? record.timeSeriesDaily[record.timeSeriesDaily.length - 1].value : 0; record.timeSeriesMonthly = monthlyTotals; record.state = record['Province/State']; record.country = record['Country/Region']; record.lat = record.Lat; record.long = record.Long; record.countrySafeName = toSafeName(record.country); record.stateSafeName = toSafeName(record.state); delete record['Province/State']; delete record['Country/Region']; delete record.Lat; delete record.Long; }); tsGlobalRecords.sort((a, b) => { if (a.country === b.country) { return a.state.localeCompare(b.state); } return a.country.localeCompare(b.country); }); const perCountryTotals = {}; tsGlobalRecords.forEach((record) => { perCountryTotals[record.country] = perCountryTotals[record.country] || { total: 0, timeSeriesDaily: {}, timeSeriesMonthly: {}, states: [], safeName: record.countrySafeName, }; const item = perCountryTotals[record.country]; item.total += record.total; item.states.push(record); record.timeSeriesDaily.forEach((ts) => { item.timeSeriesDaily[ts.key] = item.timeSeriesDaily[ts.key] || { value: 0, delta: 0, }; item.timeSeriesDaily[ts.key].value += ts.value; item.timeSeriesDaily[ts.key].delta += ts.delta; }); record.timeSeriesMonthly.forEach((ts) => { item.timeSeriesMonthly[ts.key] = item.timeSeriesMonthly[ts.key] || { value: 0, delta: 0, }; item.timeSeriesMonthly[ts.key].value += ts.value; item.timeSeriesMonthly[ts.key].delta += ts.delta; }); }); const countryArr = Object.keys(perCountryTotals).map((countryName) => { const item = perCountryTotals[countryName]; return { name: countryName, safeName: item.safeName, total: item.total, states: item.states, timeSeriesDaily: Object.keys(item.timeSeriesDaily).sort().map((date) => { return { key: date, value: item.timeSeriesDaily[date].value, delta: item.timeSeriesDaily[date].delta, }; }), timeSeriesMonthly: Object.keys(item.timeSeriesMonthly).sort().map((date) => { return { key: date, value: item.timeSeriesMonthly[date].value, delta: item.timeSeriesMonthly[date].delta, }; }), }; }); console.log(`transformed data in ${Date.now() - start}ms`); start = Date.now(); const allTmpl = path.join(templatesDir, 'all.pug'); const allHtml = pug.renderFile(allTmpl, { data: countryArr, }); const targetFile = path.join(publicDir, 'index.html'); fs.writeFileSync(targetFile, allHtml); console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`); // fs.writeFileSync(path.join(publicDir, 'countries.json'), JSON.stringify(countryArr, null, ' ')); const singleCountryTmpl = path.join(templatesDir, 'country.pug'); await Promise.all(countryArr.map(async (countryData) => { const start = Date.now(); const targetFile = path.join(publicDir, 'countries', countryData.safeName + '.html'); const countryHtml = pug.renderFile(singleCountryTmpl, { data: countryData, }); console.log(`writing to ${targetFile}`); await promiseMe(callback => fs.writeFile(targetFile, countryHtml, callback)); console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`); })); console.log(`finished in ${((Date.now() - globalStart) / 1000).toFixed(2)}s`); }; processGlobalDeaths() .then(() => { console.log('all done'); }) .catch((err) => { console.log(err); process.exit(1); });