760 lines
24 KiB
JavaScript
Executable File
760 lines
24 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const parseCsv = require('csv-parse/lib/sync');
|
|
const pug = require('pug');
|
|
const {execSync} = require('child_process');
|
|
|
|
const publicDir = path.join(__dirname, 'public');
|
|
const templatesDir = path.join(__dirname, 'tmpl');
|
|
const dataDir = path.join(__dirname, 'data');
|
|
const covidGitDir = path.join(__dirname, 'COVID-19');
|
|
const covidDataDir = path.join(covidGitDir, 'csse_covid_19_data');
|
|
const timeSeriesDir = path.join(covidDataDir, 'csse_covid_19_time_series');
|
|
|
|
const promiseMe = (fn) => {
|
|
return new Promise((resolve, reject) => {
|
|
fn((err, result) => {
|
|
if (err) {
|
|
reject(err);
|
|
return;
|
|
}
|
|
|
|
resolve(result);
|
|
});
|
|
});
|
|
};
|
|
|
|
fs.mkdirSync(path.join(__dirname, 'public', 'countries'), {
|
|
recursive: true,
|
|
});
|
|
|
|
fs.copyFileSync(
|
|
path.join(__dirname, 'node_modules', 'chart.js', 'dist', 'Chart.bundle.min.js'),
|
|
path.join(publicDir, 'Chart.bundle.js'),
|
|
);
|
|
|
|
fs.copyFileSync(
|
|
path.join(__dirname, 'node_modules', 'bootstrap', 'dist', 'css', 'bootstrap.css'),
|
|
path.join(publicDir, 'bootstrap.css'),
|
|
);
|
|
|
|
const deathsGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_global.csv');
|
|
const confirmedGlobalCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_global.csv');
|
|
const deathsUSCsv = path.join(timeSeriesDir, 'time_series_covid19_deaths_US.csv');
|
|
const confirmedUSCsv = path.join(timeSeriesDir, 'time_series_covid19_confirmed_US.csv');
|
|
const populationStatesCsv = path.join(dataDir, 'SCPRC-EST2019-18+POP-RES.csv');
|
|
const populationCountriesCsv = path.join(dataDir, 'population-world-wikipedia.tsv');
|
|
|
|
const lastGlobalDeathsUpdate = execSync(`git -C "${covidGitDir}" log -n 1 --pretty=format:'%ci' "${deathsGlobalCsv}"`, {
|
|
encoding: 'utf8',
|
|
});
|
|
const lastUSDeathsUpdate = execSync(`git -C "${covidGitDir}" log -n 1 --pretty=format:'%ci' "${deathsUSCsv}"`, {
|
|
encoding: 'utf8',
|
|
});
|
|
|
|
const lastUpdate = new Date(lastGlobalDeathsUpdate > lastUSDeathsUpdate ?
|
|
lastGlobalDeathsUpdate :
|
|
lastUSDeathsUpdate
|
|
);
|
|
|
|
const zeroPad = value => value < 10 ? `0${value}` : value.toString();
|
|
const toSafeName = x => x.replace(/[^A-Za-z]/g, '-').toLowerCase();
|
|
|
|
const processRecords = async () => {
|
|
const globalStart = Date.now();
|
|
let start = Date.now();
|
|
|
|
const timeSeriesGlobalRaw = fs.readFileSync(deathsGlobalCsv, {encoding: 'utf8'});
|
|
console.log(`read global deaths CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
const timeSeriesUSRaw = fs.readFileSync(deathsUSCsv, { encoding: 'utf8' });
|
|
console.log(`read US deaths CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
const populationUSRaw = fs.readFileSync(populationStatesCsv, {encoding: 'utf8'});
|
|
console.log(`read US states population CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
const populationCountriesRaw = fs.readFileSync(populationCountriesCsv, {encoding: 'utf8'});
|
|
console.log(`read countries population CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
const casesGlobalRaw = fs.readFileSync(confirmedGlobalCsv, {encoding: 'utf8'});
|
|
console.log(`read global confirmed CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
const casesUSRaw = fs.readFileSync(confirmedUSCsv, {encoding: 'utf8'});
|
|
console.log(`read US confirmed CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
let tsGlobalRecords = parseCsv(timeSeriesGlobalRaw, {
|
|
cast: true,
|
|
columns: true,
|
|
});
|
|
console.log(`parsed global deaths CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
let tsUSRecords = parseCsv(timeSeriesUSRaw, {
|
|
cast: true,
|
|
columns: true,
|
|
});
|
|
console.log(`parsed US deaths CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
let tsCasesGlobal = parseCsv(casesGlobalRaw, {
|
|
cast: true,
|
|
columns: true,
|
|
});
|
|
console.log(`parsed global cases CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
let tsCasesUS = parseCsv(casesUSRaw, {
|
|
cast: true,
|
|
columns: true,
|
|
});
|
|
console.log(`parsed US cases CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
let populationUSStateRecords = parseCsv(populationUSRaw, {
|
|
cast: true,
|
|
columns: true,
|
|
});
|
|
console.log(`parsed US states population CSV in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
let populationCountriesRecords = parseCsv(populationCountriesRaw, {
|
|
cast: true,
|
|
columns: true,
|
|
delimiter: '\t',
|
|
ltrim: true,
|
|
rtrim: true,
|
|
});
|
|
console.log(`parsed countries population CSV in ${Date.now() - start}ms`);
|
|
|
|
const statePopulationMap = populationUSStateRecords
|
|
.sort((a, b) => a.NAME.localeCompare(b.NAME))
|
|
.reduce((map, item) => {
|
|
map[item.NAME] = Number(item.POPESTIMATE2019);
|
|
return map;
|
|
}, {});
|
|
|
|
const countryNameMap = {
|
|
'Cape Verde': 'Cabo Verde',
|
|
'DR Congo': 'Congo (Brazzaville)',
|
|
'Congo': 'Congo (Kinshasa)',
|
|
'Ivory Coast': 'Cote d\'Ivoire',
|
|
'Czech Republic': 'Czechia',
|
|
'Vatican City': 'Holy See',
|
|
'South Korea': 'Korea, South',
|
|
'Taiwan': 'Taiwan*',
|
|
'East Timor': 'Timor-Leste',
|
|
'Palestine': 'West Bank and Gaza',
|
|
'Myanmar': 'Burma',
|
|
'São Tomé and Príncipe': 'Sao Tome and Principe',
|
|
};
|
|
|
|
const countryPopulationMap = populationCountriesRecords
|
|
.sort((a, b) => a.Country.localeCompare(b.Country))
|
|
.reduce((map, item) => {
|
|
map[item.Country] = Number(item.Pop_2019.toString().replace(/,/g, ''));
|
|
const otherName = countryNameMap[item.Country];
|
|
if (otherName) {
|
|
map[otherName] = map[item.Country];
|
|
}
|
|
return map;
|
|
}, {});
|
|
|
|
countryPopulationMap['Diamond Princess'] = statePopulationMap['Diamond Princess'] = 3711;
|
|
countryPopulationMap['MS Zaandam'] = 1829;
|
|
countryPopulationMap['Kosovo'] = 1810463;
|
|
statePopulationMap['Grand Princess'] = 3533;
|
|
if (statePopulationMap['Puerto Rico Commonwealth']) {
|
|
statePopulationMap['Puerto Rico'] = statePopulationMap['Puerto Rico Commonwealth'];
|
|
}
|
|
|
|
// tsGlobalRecords = tsGlobalRecords.filter((record) => {
|
|
// return record['Country/Region'] === 'US';
|
|
// });
|
|
|
|
const getRollingAverage = (item) => {
|
|
return item.timeSeriesDaily.map((item, i, arr) => {
|
|
const prevValues = arr.slice(Math.max(0, i - 6), i);
|
|
let valueAverage = (item.value + prevValues.reduce((sum, next) => sum + next.value, 0)) / (1 + prevValues.length);
|
|
let deltaAverage = (item.delta + prevValues.reduce((sum, next) => sum + next.delta, 0)) / (1 + prevValues.length);
|
|
|
|
if (valueAverage < 1) {
|
|
valueAverage = Math.round(valueAverage);
|
|
}
|
|
if (deltaAverage < 1) {
|
|
deltaAverage = Math.round(deltaAverage);
|
|
}
|
|
|
|
return {
|
|
key: item.key,
|
|
value: Math.round(valueAverage * 10) / 10,
|
|
delta: Math.round(deltaAverage * 10) / 10,
|
|
};
|
|
});
|
|
};
|
|
|
|
const getDoublingTime = (item) => {
|
|
return item.timeSeriesDaily.map((item, i, arr) => {
|
|
let value;
|
|
if (item.value < 10) {
|
|
value = 0;
|
|
} else {
|
|
const growthRate = calcGrowthRate(arr, i, 7);
|
|
if (growthRate <= 0 || !growthRate) {
|
|
value = 0;
|
|
} else {
|
|
value = (Math.log(2) / Math.log(1 + growthRate));
|
|
}
|
|
}
|
|
|
|
return {
|
|
key: item.key,
|
|
value: Math.round(value * 10) / 10,
|
|
};
|
|
});
|
|
};
|
|
|
|
const calcGrowthRate = (ts, i, range) => {
|
|
const items = ts.slice(Math.max(0, i - range), i + 1);
|
|
const len = items.length;
|
|
if (len < 2) {
|
|
return 0;
|
|
}
|
|
const latest = len - 1;
|
|
const earliest = 0;
|
|
const pow = 1 / (latest - earliest + 1);
|
|
const hi = items[latest].value;
|
|
const lo = Math.max(items[earliest].value, 0.5);
|
|
if (hi === 0 && lo < 1) {
|
|
return 0;
|
|
}
|
|
return Math.pow((hi / lo), pow) - 1;
|
|
};
|
|
|
|
const getGrowthRate = (ts) => {
|
|
return calcGrowthRate(ts, ts.length - 1, 7);
|
|
};
|
|
|
|
const convertTsObjectToArray = (obj) => {
|
|
return Object.keys(obj).sort().map((date) => {
|
|
return {
|
|
key: date,
|
|
value: obj[date].value,
|
|
delta: obj[date].delta,
|
|
};
|
|
})
|
|
};
|
|
|
|
const mergeTsArrayIntoObject = (source, target) => {
|
|
source.forEach((ts) => {
|
|
target[ts.key] = target[ts.key] || {
|
|
value: 0,
|
|
delta: 0,
|
|
};
|
|
target[ts.key].value += ts.value;
|
|
target[ts.key].delta += ts.delta;
|
|
});
|
|
};
|
|
|
|
const normalizeRecord = (record) => {
|
|
record.timeSeriesDaily = [];
|
|
record.timeSeriesMonthly = [];
|
|
const dateColumns = Object.keys(record).filter(x => /^\d+\/\d+\/\d+$/.test(x))
|
|
.map(key => {
|
|
return {
|
|
key,
|
|
date: new Date(key),
|
|
};
|
|
})
|
|
.sort((a, b) => {
|
|
if (a.date.getTime() === b.date.getTime()) {
|
|
return 0;
|
|
}
|
|
|
|
return a.date.getTime() < b.date.getTime() ? -1 : 1;
|
|
});
|
|
|
|
const toSortableDate = date => [
|
|
date.getUTCFullYear(),
|
|
zeroPad(date.getUTCMonth() + 1),
|
|
zeroPad(date.getUTCDate()),
|
|
].join('-');
|
|
|
|
dateColumns.forEach((obj) => {
|
|
const value = Number(record[obj.key]) || 0;
|
|
const date = obj.date;
|
|
delete record[obj.key];
|
|
const sortableKey = toSortableDate(date);
|
|
const lastItem = record.timeSeriesDaily[record.timeSeriesDaily.length - 1];
|
|
record.timeSeriesDaily.push({
|
|
key: sortableKey,
|
|
value,
|
|
delta: lastItem ? value - lastItem.value : 0,
|
|
});
|
|
});
|
|
|
|
const monthlyTotals = [];
|
|
const monthlyMaxes = {};
|
|
record.timeSeriesDaily.forEach((item) => {
|
|
const key = item.key.replace(/-\d+$/, '');
|
|
if (!(key in monthlyMaxes)) {
|
|
monthlyMaxes[key] = {
|
|
date: item.key,
|
|
value: item.value,
|
|
};
|
|
} else if (item.key > monthlyMaxes[key].date) {
|
|
monthlyMaxes[key] = {
|
|
date: item.key,
|
|
value: item.value,
|
|
}
|
|
}
|
|
});
|
|
|
|
Object.keys(monthlyMaxes).forEach((key) => {
|
|
monthlyTotals.push({
|
|
key,
|
|
value: monthlyMaxes[key].value,
|
|
});
|
|
});
|
|
|
|
monthlyTotals.sort((a, b) => a.key.localeCompare(b.key));
|
|
monthlyTotals.forEach((item, i) => {
|
|
const prev = monthlyTotals[i - 1];
|
|
item.delta = prev ? item.value - prev.value : 0;
|
|
});
|
|
|
|
record.total = record.timeSeriesDaily.length ?
|
|
record.timeSeriesDaily[record.timeSeriesDaily.length - 1].value :
|
|
0;
|
|
record.timeSeriesMonthly = monthlyTotals;
|
|
record.rollingAverageDaily = getRollingAverage(record);
|
|
|
|
record.state = record['Province/State'];
|
|
record.country = record['Country/Region'];
|
|
record.lat = record.Lat;
|
|
record.long = record.Long;
|
|
record.county = record.Admin2 || '';
|
|
record.population = record.Population || null;
|
|
|
|
if (record.country === 'US') {
|
|
record.country = 'United States';
|
|
}
|
|
|
|
record.countrySafeName = toSafeName(record.country);
|
|
record.stateSafeName = toSafeName(record.state);
|
|
record.countySafeName = toSafeName(record.county);
|
|
|
|
delete record['Province/State'];
|
|
delete record['Country/Region'];
|
|
delete record.Lat;
|
|
delete record.Long;
|
|
delete record.Admin2;
|
|
delete record.Population;
|
|
};
|
|
|
|
const getRecordKey = record => `${record.country || ''}:${record.state || ''}:${record.county || ''}`;
|
|
|
|
// pre-process confirmed case data for later lookup
|
|
tsCasesUS.forEach((usRecord) => {
|
|
const newRecord = {
|
|
...usRecord,
|
|
needsRollup: false,
|
|
Long: usRecord.Long_,
|
|
'Province/State': usRecord.Province_State,
|
|
'Country/Region': usRecord.Country_Region,
|
|
}
|
|
|
|
delete newRecord.UID;
|
|
delete newRecord.iso2;
|
|
delete newRecord.iso3;
|
|
delete newRecord.code3;
|
|
delete newRecord.FIPS;
|
|
delete newRecord.Combined_Key;
|
|
delete newRecord.Long_;
|
|
delete newRecord.Province_State;
|
|
delete newRecord.Country_Region;
|
|
|
|
tsCasesGlobal.push(newRecord);
|
|
});
|
|
|
|
const confirmedCasesLookup = tsCasesGlobal.reduce((lookup, record) => {
|
|
normalizeRecord(record);
|
|
const key = getRecordKey(record);
|
|
if (lookup[key]) {
|
|
throw new Error(`key "${key}" already exists in confirmed case lookup table`);
|
|
}
|
|
lookup[key] = record;
|
|
return lookup;
|
|
}, {});
|
|
|
|
// state/county data is separated for the US and doesn't need to be rolled up
|
|
tsUSRecords.forEach((usRecord) => {
|
|
const newRecord = {
|
|
...usRecord,
|
|
needsRollup: false,
|
|
Long: usRecord.Long_,
|
|
'Province/State': usRecord.Province_State,
|
|
'Country/Region': usRecord.Country_Region,
|
|
};
|
|
|
|
delete newRecord.UID;
|
|
delete newRecord.iso2;
|
|
delete newRecord.iso3;
|
|
delete newRecord.code3;
|
|
delete newRecord.FIPS;
|
|
delete newRecord.Combined_Key;
|
|
delete newRecord.Long_;
|
|
delete newRecord.Province_State;
|
|
delete newRecord.Country_Region;
|
|
|
|
tsGlobalRecords.push(newRecord);
|
|
});
|
|
|
|
start = Date.now();
|
|
tsGlobalRecords.forEach((record) => {
|
|
normalizeRecord(record);
|
|
|
|
const recordKey = getRecordKey(record);
|
|
const confirmedCases = confirmedCasesLookup[recordKey];
|
|
if (!confirmedCases) {
|
|
throw new Error(`no cases found in lookup for key "${recordKey}"`);
|
|
}
|
|
|
|
record.cases = {
|
|
total: confirmedCases.timeSeriesDaily[confirmedCases.timeSeriesDaily.length - 1].value,
|
|
timeSeriesDaily: confirmedCases.timeSeriesDaily,
|
|
timeSeriesMonthly: confirmedCases.timeSeriesMonthly,
|
|
rollingAverageDaily: confirmedCases.rollingAverageDaily,
|
|
};
|
|
|
|
if (!record.population && !record.state && !record.county) {
|
|
const mappedPop = countryPopulationMap[record.country];
|
|
if (!mappedPop) {
|
|
console.log(require('util').inspect(countryPopulationMap, false, null, true));
|
|
throw new Error('found no population for ' + record.country);
|
|
}
|
|
record.population = mappedPop;
|
|
} else if (!record.population && record.state && !record.county && record.country === 'United States') {
|
|
// US territories and cruise ships
|
|
const mappedPop = statePopulationMap[record.state];
|
|
if (!mappedPop) {
|
|
console.log(require('util').inspect(statePopulationMap, false, null, true));
|
|
throw new Error('found no population for ' + record.state);
|
|
}
|
|
// console.log(`setting population to ${mappedPop} for ${record.state}`);
|
|
record.population = mappedPop;
|
|
}
|
|
|
|
record.deathsPerMillion = !!record.population ? record.total / record.population * 1000000 : 0;
|
|
record.casesPerMillion = !!record.population ? record.cases.total / record.population * 1000000 : 0;
|
|
record.deathGrowthRate = getGrowthRate(record.timeSeriesDaily);
|
|
record.caseGrowthRate = getGrowthRate(record.cases.timeSeriesDaily);
|
|
record.caseFatalityRate = record.cases.total !== 0 ? record.total / record.cases.total : 0;
|
|
});
|
|
|
|
tsGlobalRecords.sort((a, b) => {
|
|
if (a.country === b.country) {
|
|
return a.state.localeCompare(b.state);
|
|
}
|
|
|
|
return a.country.localeCompare(b.country);
|
|
});
|
|
|
|
const perCountryTotals = {};
|
|
const perStateTotals = {};
|
|
tsGlobalRecords.forEach((record) => {
|
|
perCountryTotals[record.country] = perCountryTotals[record.country] || {
|
|
total: 0,
|
|
population: 0,
|
|
timeSeriesDaily: {},
|
|
timeSeriesMonthly: {},
|
|
deathsPerMillion: 0,
|
|
casesPerMillion: 0,
|
|
states: [],
|
|
safeName: record.countrySafeName,
|
|
cases: {
|
|
total: 0,
|
|
timeSeriesDaily: {},
|
|
timeSeriesMonthly: {},
|
|
},
|
|
};
|
|
|
|
const item = perCountryTotals[record.country];
|
|
if (!record.state && !record.county) {
|
|
// country population
|
|
item.population = record.population;
|
|
}
|
|
|
|
if (record.county) {
|
|
record.name = record.county;
|
|
record.safeName = toSafeName(record.county);
|
|
} else if (record.state) {
|
|
record.name = record.state;
|
|
record.safeName = toSafeName(record.state);
|
|
}
|
|
|
|
// roll up up state/county data
|
|
if (record.state && record.county) {
|
|
const stateItem = perStateTotals[record.state] = perStateTotals[record.state] || {
|
|
name: record.state,
|
|
safeName: record.stateSafeName,
|
|
country: record.country,
|
|
countrySafeName: record.countrySafeName,
|
|
total: 0,
|
|
population: 0,
|
|
deathsPerMillion: 0,
|
|
casesPerMillion: 0,
|
|
timeSeriesDaily: {},
|
|
timeSeriesMonthly: {},
|
|
cases: {
|
|
total: 0,
|
|
timeSeriesDaily: {},
|
|
timeSeriesMonthly: {},
|
|
},
|
|
counties: [],
|
|
};
|
|
|
|
if (!stateItem.population && record.country === 'United States') {
|
|
const population = statePopulationMap[record.state];
|
|
if (!population) {
|
|
console.log(require('util').inspect(statePopulationMap, false, null, true));
|
|
throw new Error('no population for ' + record.state);
|
|
}
|
|
|
|
stateItem.population = population;
|
|
}
|
|
|
|
stateItem.total += record.total;
|
|
stateItem.cases.total += record.cases.total;
|
|
|
|
mergeTsArrayIntoObject(record.timeSeriesDaily, stateItem.timeSeriesDaily);
|
|
mergeTsArrayIntoObject(record.timeSeriesMonthly, stateItem.timeSeriesMonthly);
|
|
mergeTsArrayIntoObject(record.cases.timeSeriesDaily, stateItem.cases.timeSeriesDaily);
|
|
mergeTsArrayIntoObject(record.cases.timeSeriesMonthly, stateItem.cases.timeSeriesMonthly);
|
|
|
|
stateItem.counties.push(record);
|
|
} else {
|
|
item.states.push(record);
|
|
}
|
|
|
|
if (record.needsRollup === false) {
|
|
return;
|
|
}
|
|
|
|
item.total += record.total;
|
|
item.cases.total += record.cases.total;
|
|
|
|
mergeTsArrayIntoObject(record.timeSeriesDaily, item.timeSeriesDaily);
|
|
mergeTsArrayIntoObject(record.timeSeriesMonthly, item.timeSeriesMonthly);
|
|
mergeTsArrayIntoObject(record.cases.timeSeriesDaily, item.cases.timeSeriesDaily);
|
|
mergeTsArrayIntoObject(record.cases.timeSeriesMonthly, item.cases.timeSeriesMonthly);
|
|
});
|
|
|
|
Object.keys(perStateTotals).forEach((stateName) => {
|
|
const item = perStateTotals[stateName];
|
|
if (!item.cases) {
|
|
throw new Error('no cases');
|
|
}
|
|
const stateItem = {
|
|
name: stateName,
|
|
safeName: item.safeName,
|
|
country: item.country,
|
|
countrySafeName: item.countrySafeName,
|
|
total: item.total,
|
|
counties: item.counties,
|
|
population: item.population,
|
|
deathsPerMillion: item.population > 0 ? item.total / item.population * 1000000 : 0,
|
|
casesPerMillion: item.population > 0 ? item.cases.total / item.population * 1000000 : 0,
|
|
timeSeriesDaily: convertTsObjectToArray(item.timeSeriesDaily),
|
|
timeSeriesMonthly: convertTsObjectToArray(item.timeSeriesMonthly),
|
|
cases: {
|
|
total: item.cases.total,
|
|
timeSeriesDaily: convertTsObjectToArray(item.cases.timeSeriesDaily),
|
|
timeSeriesMonthly: convertTsObjectToArray(item.cases.timeSeriesMonthly),
|
|
},
|
|
caseFatalityRate: item.cases.total !== 0 ? item.total / item.cases.total : 0,
|
|
};
|
|
|
|
stateItem.deathGrowthRate = getGrowthRate(stateItem.timeSeriesDaily);
|
|
stateItem.caseGrowthRate = getGrowthRate(stateItem.cases.timeSeriesDaily);
|
|
stateItem.rollingAverageDaily = getRollingAverage(stateItem);
|
|
stateItem.doublingDaily = getDoublingTime(stateItem);
|
|
stateItem.cases.rollingAverageDaily = getRollingAverage(stateItem.cases);
|
|
stateItem.cases.doublingDaily = getDoublingTime(stateItem.cases);
|
|
|
|
// insert into states array for the country
|
|
perCountryTotals[item.country].states.push(stateItem);
|
|
});
|
|
|
|
const countryArr = Object.keys(perCountryTotals).map((countryName) => {
|
|
const item = perCountryTotals[countryName];
|
|
if (!item.population) {
|
|
// some countries don't have roll up (e.g. Canada, China, Australia)
|
|
// and only contain province data instead of data for the whole country.
|
|
// population doesn't get set in those cases.
|
|
item.population = countryPopulationMap[countryName];
|
|
}
|
|
|
|
if (!item.cases) {
|
|
throw new Error('no cases for country');
|
|
}
|
|
|
|
const countryItem = {
|
|
name: countryName,
|
|
safeName: item.safeName,
|
|
total: item.total,
|
|
states: item.states,
|
|
population: item.population,
|
|
deathsPerMillion: item.population > 0 ? item.total / item.population * 1000000 : 0,
|
|
casesPerMillion: item.population > 0 ? item.cases.total / item.population * 1000000 : 0,
|
|
timeSeriesDaily: convertTsObjectToArray(item.timeSeriesDaily),
|
|
timeSeriesMonthly: convertTsObjectToArray(item.timeSeriesMonthly),
|
|
cases: {
|
|
total: item.cases.total,
|
|
timeSeriesDaily: convertTsObjectToArray(item.cases.timeSeriesDaily),
|
|
timeSeriesMonthly: convertTsObjectToArray(item.cases.timeSeriesMonthly),
|
|
},
|
|
caseFatalityRate: item.cases.total !== 0 ? item.total / item.cases.total : 0,
|
|
};
|
|
|
|
countryItem.deathGrowthRate = getGrowthRate(countryItem.timeSeriesDaily);
|
|
countryItem.caseGrowthRate = getGrowthRate(countryItem.cases.timeSeriesDaily);
|
|
countryItem.rollingAverageDaily = getRollingAverage(countryItem);
|
|
countryItem.doublingDaily = getDoublingTime(countryItem);
|
|
countryItem.cases.rollingAverageDaily = getRollingAverage(countryItem.cases);
|
|
countryItem.cases.doublingDaily = getDoublingTime(countryItem.cases);
|
|
return countryItem;
|
|
});
|
|
|
|
const worldData = {
|
|
name: 'Worldwide',
|
|
safeName: 'worldwide',
|
|
total: 0,
|
|
countries: countryArr,
|
|
timeSeriesDaily: {},
|
|
timeSeriesMonthly: {},
|
|
cases: {
|
|
total: 0,
|
|
timeSeriesDaily: {},
|
|
timeSeriesMonthly: {},
|
|
},
|
|
};
|
|
|
|
countryArr.forEach((countryData) => {
|
|
worldData.total += countryData.total;
|
|
worldData.cases.total += countryData.cases.total;
|
|
|
|
mergeTsArrayIntoObject(countryData.timeSeriesDaily, worldData.timeSeriesDaily);
|
|
mergeTsArrayIntoObject(countryData.timeSeriesMonthly, worldData.timeSeriesMonthly);
|
|
mergeTsArrayIntoObject(countryData.cases.timeSeriesDaily, worldData.cases.timeSeriesDaily);
|
|
mergeTsArrayIntoObject(countryData.cases.timeSeriesMonthly, worldData.cases.timeSeriesMonthly);
|
|
});
|
|
|
|
worldData.timeSeriesDaily = convertTsObjectToArray(worldData.timeSeriesDaily);
|
|
worldData.timeSeriesMonthly = convertTsObjectToArray(worldData.timeSeriesMonthly);
|
|
worldData.cases.timeSeriesDaily = convertTsObjectToArray(worldData.cases.timeSeriesDaily);
|
|
worldData.cases.timeSeriesMonthly = convertTsObjectToArray(worldData.cases.timeSeriesMonthly);
|
|
|
|
worldData.deathGrowthRate = getGrowthRate(worldData.timeSeriesDaily);
|
|
worldData.caseGrowthRate = getGrowthRate(worldData.cases.timeSeriesDaily);
|
|
worldData.rollingAverageDaily = getRollingAverage(worldData);
|
|
worldData.doublingDaily = getDoublingTime(worldData);
|
|
worldData.cases.rollingAverageDaily = getRollingAverage(worldData.cases);
|
|
worldData.cases.doublingDaily = getDoublingTime(worldData.cases);
|
|
|
|
worldData.population = 7781841000;
|
|
worldData.deathsPerMillion = worldData.total / worldData.population * 1000000;
|
|
worldData.casesPerMillion = worldData.cases.total / worldData.population * 1000000;
|
|
worldData.caseFatalityRate = worldData.cases.total !== 0 ? worldData.total / worldData.cases.total : 0;
|
|
|
|
console.log(`transformed data in ${Date.now() - start}ms`);
|
|
|
|
start = Date.now();
|
|
const worldTmpl = path.join(templatesDir, 'world.pug');
|
|
const worldHtml = pug.renderFile(worldTmpl, {
|
|
data: worldData,
|
|
$title: 'Worldwide',
|
|
lastUpdate,
|
|
});
|
|
|
|
const targetFile = path.join(publicDir, 'index.html');
|
|
fs.writeFileSync(targetFile, worldHtml);
|
|
console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);
|
|
|
|
const singleCountryTmpl = path.join(templatesDir, 'country.pug');
|
|
const singleStateTmpl = path.join(templatesDir, 'state.pug');
|
|
const singleCountyTmpl = path.join(templatesDir, 'county.pug');
|
|
|
|
const countryFn = pug.compileFile(singleCountryTmpl);
|
|
const stateFn = pug.compileFile(singleStateTmpl);
|
|
const countyFn = pug.compileFile(singleCountyTmpl);
|
|
|
|
for (const countryData of countryArr) {
|
|
const start = Date.now();
|
|
const targetFile = path.join(publicDir, 'countries', countryData.safeName + '.html');
|
|
const countryHtml = countryFn({
|
|
data: countryData,
|
|
$title: countryData.name,
|
|
lastUpdate,
|
|
});
|
|
await promiseMe(callback => fs.writeFile(targetFile, countryHtml, callback));
|
|
console.log(`[country:${countryData.name}] wrote to ${targetFile} in ${Date.now() - start}ms`);
|
|
|
|
if (!countryData.states) {
|
|
continue;
|
|
}
|
|
|
|
for (const stateData of countryData.states) {
|
|
if (!stateData.name || !stateData.counties || !stateData.counties.length) {
|
|
continue;
|
|
}
|
|
|
|
const start = Date.now();
|
|
const targetFile = path.join(publicDir, 'countries', countryData.safeName + '-' + stateData.safeName + '.html');
|
|
const stateHtml = stateFn({
|
|
data: stateData,
|
|
$title: stateData.name + ', ' + countryData.name,
|
|
lastUpdate,
|
|
});
|
|
await promiseMe(callback => fs.writeFile(targetFile, stateHtml, callback));
|
|
console.log(`[state:${stateData.name}] wrote to ${targetFile} in ${Date.now() - start}ms`);
|
|
|
|
for (const countyData of stateData.counties) {
|
|
countyData.population = countyData.population || 0;
|
|
|
|
const start = Date.now();
|
|
const targetFile = path.join(
|
|
publicDir,
|
|
'countries',
|
|
`${countryData.safeName}-${stateData.safeName}-${countyData.safeName}.html`,
|
|
);
|
|
const countyHtml = countyFn({
|
|
data: countyData,
|
|
$title: `${countyData.name}, ${countyData.state}, ${countyData.country}`,
|
|
lastUpdate,
|
|
});
|
|
await promiseMe(callback => fs.writeFile(targetFile, countyHtml, callback));
|
|
console.log(`[county:${countyData.state}-${countyData.name}] wrote to ${targetFile} in ${Date.now() - start}ms`);
|
|
}
|
|
}
|
|
}
|
|
|
|
console.log(`finished in ${((Date.now() - globalStart) / 1000).toFixed(2)}s`);
|
|
};
|
|
|
|
processRecords()
|
|
.then(() => {
|
|
console.log('all done');
|
|
})
|
|
.catch((err) => {
|
|
console.log(err);
|
|
process.exit(1);
|
|
});
|