diff --git a/generate.js b/generate.js index a0efe4e..d580f7d 100755 --- a/generate.js +++ b/generate.js @@ -62,7 +62,7 @@ const lastUpdate = new Date(lastGlobalDeathsUpdate > lastUSDeathsUpdate ? const zeroPad = value => value < 10 ? `0${value}` : value.toString(); const toSafeName = x => x.replace(/[^A-Za-z]/g, '-').toLowerCase(); -const processGlobalDeaths = async () => { +const processRecords = async () => { const globalStart = Date.now(); let start = Date.now(); @@ -81,6 +81,14 @@ const processGlobalDeaths = async () => { const populationCountriesRaw = fs.readFileSync(populationCountriesCsv, {encoding: 'utf8'}); console.log(`read countries population CSV in ${Date.now() - start}ms`); + start = Date.now(); + const casesGlobalRaw = fs.readFileSync(confirmedGlobalCsv, {encoding: 'utf8'}); + console.log(`read global confirmed CSV in ${Date.now() - start}ms`); + + start = Date.now(); + const casesUSRaw = fs.readFileSync(confirmedUSCsv, {encoding: 'utf8'}); + console.log(`read US confirmed CSV in ${Date.now() - start}ms`); + start = Date.now(); let tsGlobalRecords = parseCsv(timeSeriesGlobalRaw, { cast: true, @@ -95,6 +103,20 @@ const processGlobalDeaths = async () => { }); console.log(`parsed US deaths CSV in ${Date.now() - start}ms`); + start = Date.now(); + let tsCasesGlobal = parseCsv(casesGlobalRaw, { + cast: true, + columns: true, + }); + console.log(`parsed global cases CSV in ${Date.now() - start}ms`); + + start = Date.now(); + let tsCasesUS = parseCsv(casesUSRaw, { + cast: true, + columns: true, + }); + console.log(`parsed US cases CSV in ${Date.now() - start}ms`); + start = Date.now(); let populationUSStateRecords = parseCsv(populationUSRaw, { cast: true, @@ -217,31 +239,7 @@ const processGlobalDeaths = async () => { return calcGrowthRate(record.timeSeriesDaily, record.timeSeriesDaily.length - 1, 7); }; - // state/county data is separated for the US and doesn't need to be rolled up - tsUSRecords.forEach((usRecord) => { - const newRecord = { - ...usRecord, - needsRollup: false, - Long: usRecord.Long_, - 'Province/State': usRecord.Province_State, - 'Country/Region': usRecord.Country_Region, - }; - - delete newRecord.UID; - delete newRecord.iso2; - delete newRecord.iso3; - delete newRecord.code3; - delete newRecord.FIPS; - delete newRecord.Combined_Key; - delete newRecord.Long_; - delete newRecord.Province_State; - delete newRecord.Country_Region; - - tsGlobalRecords.push(newRecord); - }); - - start = Date.now(); - tsGlobalRecords.forEach((record) => { + const normalizeRecord = (record) => { record.timeSeriesDaily = []; record.timeSeriesMonthly = []; const dateColumns = Object.keys(record).filter(x => /^\d+\/\d+\/\d+$/.test(x)) @@ -335,6 +333,81 @@ const processGlobalDeaths = async () => { delete record.Long; delete record.Admin2; delete record.Population; + }; + + const getRecordKey = record => `${record.country || ''}:${record.state || ''}:${record.county || ''}`; + + // pre-process confirmed case data for later lookup + tsCasesUS.forEach((usRecord) => { + const newRecord = { + ...usRecord, + needsRollup: false, + Long: usRecord.Long_, + 'Province/State': usRecord.Province_State, + 'Country/Region': usRecord.Country_Region, + } + + delete newRecord.UID; + delete newRecord.iso2; + delete newRecord.iso3; + delete newRecord.code3; + delete newRecord.FIPS; + delete newRecord.Combined_Key; + delete newRecord.Long_; + delete newRecord.Province_State; + delete newRecord.Country_Region; + + tsCasesGlobal.push(newRecord); + }); + + const confirmedCasesLookup = tsCasesGlobal.reduce((lookup, record) => { + normalizeRecord(record); + const key = getRecordKey(record); + if (lookup[key]) { + throw new Error(`key "${key}" already exists in confirmed case lookup table`); + } + lookup[key] = record; + return lookup; + }, {}); + + // state/county data is separated for the US and doesn't need to be rolled up + tsUSRecords.forEach((usRecord) => { + const newRecord = { + ...usRecord, + needsRollup: false, + Long: usRecord.Long_, + 'Province/State': usRecord.Province_State, + 'Country/Region': usRecord.Country_Region, + }; + + delete newRecord.UID; + delete newRecord.iso2; + delete newRecord.iso3; + delete newRecord.code3; + delete newRecord.FIPS; + delete newRecord.Combined_Key; + delete newRecord.Long_; + delete newRecord.Province_State; + delete newRecord.Country_Region; + + tsGlobalRecords.push(newRecord); + }); + + start = Date.now(); + tsGlobalRecords.forEach((record) => { + normalizeRecord(record); + + const recordKey = getRecordKey(record); + const confirmedCases = confirmedCasesLookup[recordKey]; + if (!confirmedCases) { + throw new Error(`no cases found in lookup for key "${recordKey}"`); + } + + record.cases = { + timeSeriesDaily: confirmedCases.timeSeriesDaily, + timeSeriesMonthly: confirmedCases.timeSeriesMonthly, + rollingAverageDaily: confirmedCases.rollingAverageDaily, + }; if (!record.population && !record.state && !record.county) { const mappedPop = countryPopulationMap[record.country]; @@ -376,6 +449,10 @@ const processGlobalDeaths = async () => { timeSeriesMonthly: {}, states: [], safeName: record.countrySafeName, + cases: { + timeSeriesDaily: {}, + timeSeriesMonthly: {}, + }, }; const item = perCountryTotals[record.country]; @@ -404,6 +481,10 @@ const processGlobalDeaths = async () => { deathsPerMillion: 0, timeSeriesDaily: {}, timeSeriesMonthly: {}, + cases: { + timeSeriesDaily: {}, + timeSeriesMonthly: {}, + }, counties: [], }; @@ -436,6 +517,24 @@ const processGlobalDeaths = async () => { stateItem.timeSeriesMonthly[ts.key].delta += ts.delta; }); + record.cases.timeSeriesDaily.forEach((ts) => { + stateItem.cases.timeSeriesDaily[ts.key] = stateItem.cases.timeSeriesDaily[ts.key] || { + value: 0, + delta: 0, + }; + stateItem.cases.timeSeriesDaily[ts.key].value += ts.value; + stateItem.cases.timeSeriesDaily[ts.key].delta += ts.delta; + }); + + record.cases.timeSeriesMonthly.forEach((ts) => { + stateItem.cases.timeSeriesMonthly[ts.key] = stateItem.cases.timeSeriesMonthly[ts.key] || { + value: 0, + delta: 0, + }; + stateItem.cases.timeSeriesMonthly[ts.key].value += ts.value; + stateItem.cases.timeSeriesMonthly[ts.key].delta += ts.delta; + }); + stateItem.counties.push(record); } else { item.states.push(record); @@ -463,10 +562,31 @@ const processGlobalDeaths = async () => { item.timeSeriesMonthly[ts.key].value += ts.value; item.timeSeriesMonthly[ts.key].delta += ts.delta; }); + + record.cases.timeSeriesDaily.forEach((ts) => { + item.cases.timeSeriesDaily[ts.key] = item.cases.timeSeriesDaily[ts.key] || { + value: 0, + delta: 0, + }; + item.cases.timeSeriesDaily[ts.key].value += ts.value; + item.cases.timeSeriesDaily[ts.key].delta += ts.delta; + }); + + record.cases.timeSeriesMonthly.forEach((ts) => { + item.cases.timeSeriesMonthly[ts.key] = item.cases.timeSeriesMonthly[ts.key] || { + value: 0, + delta: 0, + }; + item.cases.timeSeriesMonthly[ts.key].value += ts.value; + item.cases.timeSeriesMonthly[ts.key].delta += ts.delta; + }); }); Object.keys(perStateTotals).forEach((stateName) => { const item = perStateTotals[stateName]; + if (!item.cases) { + throw new Error('no cases'); + } const stateItem = { name: stateName, safeName: item.safeName, @@ -490,11 +610,28 @@ const processGlobalDeaths = async () => { delta: item.timeSeriesMonthly[date].delta, }; }), + cases: { + timeSeriesDaily: Object.keys(item.cases.timeSeriesDaily).sort().map((date) => { + return { + key: date, + value: item.cases.timeSeriesDaily[date].value, + delta: item.cases.timeSeriesDaily[date].delta, + }; + }), + timeSeriesMonthly: Object.keys(item.cases.timeSeriesMonthly).sort().map((date) => { + return { + key: date, + value: item.cases.timeSeriesMonthly[date].value, + delta: item.cases.timeSeriesMonthly[date].delta, + }; + }), + }, }; stateItem.deathGrowthRate = getGrowthRate(stateItem); stateItem.rollingAverageDaily = getRollingAverage(stateItem); stateItem.doublingDaily = getDoublingTime(stateItem); + stateItem.cases.rollingAverageDaily = getRollingAverage(stateItem.cases); // insert into states array for the country perCountryTotals[item.country].states.push(stateItem); @@ -509,6 +646,10 @@ const processGlobalDeaths = async () => { item.population = countryPopulationMap[countryName]; } + if (!item.cases) { + throw new Error('no cases for country'); + } + const countryItem = { name: countryName, safeName: item.safeName, @@ -530,11 +671,28 @@ const processGlobalDeaths = async () => { delta: item.timeSeriesMonthly[date].delta, }; }), + cases: { + timeSeriesDaily: Object.keys(item.cases.timeSeriesDaily).sort().map((date) => { + return { + key: date, + value: item.cases.timeSeriesDaily[date].value, + delta: item.cases.timeSeriesDaily[date].delta, + }; + }), + timeSeriesMonthly: Object.keys(item.cases.timeSeriesMonthly).sort().map((date) => { + return { + key: date, + value: item.cases.timeSeriesMonthly[date].value, + delta: item.cases.timeSeriesMonthly[date].delta, + }; + }), + }, }; countryItem.deathGrowthRate = getGrowthRate(countryItem); countryItem.rollingAverageDaily = getRollingAverage(countryItem); countryItem.doublingDaily = getDoublingTime(countryItem); + countryItem.cases.rollingAverageDaily = getRollingAverage(countryItem.cases); return countryItem; }); @@ -545,6 +703,10 @@ const processGlobalDeaths = async () => { countries: countryArr, timeSeriesDaily: {}, timeSeriesMonthly: {}, + cases: { + timeSeriesDaily: {}, + timeSeriesMonthly: {}, + }, }; countryArr.forEach((countryData) => { @@ -567,6 +729,24 @@ const processGlobalDeaths = async () => { worldData.timeSeriesMonthly[ts.key].value += ts.value; worldData.timeSeriesMonthly[ts.key].delta += ts.delta; }); + + countryData.cases.timeSeriesDaily.forEach((ts) => { + worldData.cases.timeSeriesDaily[ts.key] = worldData.cases.timeSeriesDaily[ts.key] || { + value: 0, + delta: 0, + }; + worldData.cases.timeSeriesDaily[ts.key].value += ts.value; + worldData.cases.timeSeriesDaily[ts.key].delta += ts.delta; + }); + + countryData.cases.timeSeriesMonthly.forEach((ts) => { + worldData.cases.timeSeriesMonthly[ts.key] = worldData.cases.timeSeriesMonthly[ts.key] || { + value: 0, + delta: 0, + }; + worldData.cases.timeSeriesMonthly[ts.key].value += ts.value; + worldData.cases.timeSeriesMonthly[ts.key].delta += ts.delta; + }); }); worldData.timeSeriesDaily = Object.keys(worldData.timeSeriesDaily).sort().map((date) => { @@ -583,10 +763,25 @@ const processGlobalDeaths = async () => { delta: worldData.timeSeriesMonthly[date].delta, }; }); + worldData.cases.timeSeriesDaily = Object.keys(worldData.cases.timeSeriesDaily).sort().map((date) => { + return { + key: date, + value: worldData.cases.timeSeriesDaily[date].value, + delta: worldData.cases.timeSeriesDaily[date].delta, + }; + }); + worldData.cases.timeSeriesMonthly = Object.keys(worldData.cases.timeSeriesMonthly).sort().map((date) => { + return { + key: date, + value: worldData.cases.timeSeriesMonthly[date].value, + delta: worldData.cases.timeSeriesMonthly[date].delta, + }; + }); worldData.deathGrowthRate = getGrowthRate(worldData); worldData.rollingAverageDaily = getRollingAverage(worldData); worldData.doublingDaily = getDoublingTime(worldData); + worldData.cases.rollingAverageDaily = getRollingAverage(worldData.cases); worldData.population = 7781841000; worldData.deathsPerMillion = worldData.total / worldData.population * 1000000; @@ -604,7 +799,6 @@ const processGlobalDeaths = async () => { const targetFile = path.join(publicDir, 'index.html'); fs.writeFileSync(targetFile, worldHtml); console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`); - // fs.writeFileSync(path.join(publicDir, 'countries.json'), JSON.stringify(countryArr, null, ' ')); const singleCountryTmpl = path.join(templatesDir, 'country.pug'); const singleStateTmpl = path.join(templatesDir, 'state.pug'); @@ -645,7 +839,7 @@ const processGlobalDeaths = async () => { console.log(`finished in ${((Date.now() - globalStart) / 1000).toFixed(2)}s`); }; -processGlobalDeaths() +processRecords() .then(() => { console.log('all done'); }) diff --git a/tmpl/master.pug b/tmpl/master.pug index 5eb026c..48ee1aa 100644 --- a/tmpl/master.pug +++ b/tmpl/master.pug @@ -11,7 +11,28 @@ html vertical-align: middle !important; } th.sorted, td.sorted { - background-color: #e0eefd; + background-color: #e0eaf7; + } + .geo-bg-dark { + background-color: #8e8e8e; + color: white; + } + .cases-bg-dark { + background-color: #aaa55e; + color: white; + } + .deaths-bg-dark { + background-color: #a65353; + color: white; + } + .geo-bg { + background-color: #eeeeee; + } + .cases-bg { + background-color: #f9f6d5; + } + .deaths-bg { + background-color: #eac8c8; } .table-sm { font-size: 80%; @@ -170,9 +191,19 @@ html }); } - function makeHeroChart(id, title, labels, totalDeaths, newDeaths, rollingAverage, doubling) { + function makeHeroChart( + id, + title, + labels, + totalDeaths, + newDeaths, + rollingAverage, + doubling, + totalCases, + newCases, + ) { const canvas = document.getElementById(id); - charts.heroMaxValue = totalDeaths[totalDeaths.length - 1]; + charts.heroMaxValue = totalCases.reduce((max, value) => Math.max(max, value), 0); const firstNonZeroDeathIndex = totalDeaths.findIndex(value => value > 0); const start = Math.max(0, firstNonZeroDeathIndex - 2); @@ -182,6 +213,8 @@ html const newData = newDeaths.slice(start, end); const rollingData = rollingAverage.slice(start, end); const doublingData = doubling.slice(start, end); + const totalCaseData = totalCases.slice(start, end); + const newCaseData = newCases.slice(start, end); const months = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ]; @@ -195,40 +228,59 @@ html labels: realLabels, datasets: [ { - label: 'Cumulative', - data: totalData, - fill: '1', - borderColor: 'rgb(196, 64, 64)', + label: 'Cases', + data: totalCaseData, + fill: '2', + borderColor: 'rgb(161,150,20)', + backgroundColor: 'rgba(161,150,20, 0.25)', borderWidth: 1, - backgroundColor: 'rgba(196, 128, 128, 0.25)', }, { - label: 'New (rolling)', + label: 'New Cases', + data: newCaseData, + fill: false, + borderColor: 'rgba(206,108,46,0.5)', + backgroundColor: 'rgba(206,108,46,0.5)', + borderWidth: 1, + pointRadius: 0, + }, + { + label: 'Deaths', + data: totalData, + fill: '3', + borderColor: 'rgb(196, 64, 64)', + backgroundColor: 'rgba(196, 64, 64, 0.25)', + borderWidth: 1, + }, + { + label: 'New Deaths (rolling)', data: rollingData, fill: 'origin', borderColor: 'rgb(20,24,59)', - borderWidth: 1, backgroundColor: 'rgba(96, 96, 164, 0.25)', + borderWidth: 1, }, { - label: 'New', + label: 'New Deaths', data: newData, fill: false, borderColor: 'rgb(96, 96, 96, 0.25)', + backgroundColor: 'rgb(96, 96, 96, 0.25)', borderWidth: 1, }, { - label: 'Days to 2x', + label: 'Days to 2x deaths', data: doublingData, fill: false, borderColor: 'rgb(187,40,193, 0.5)', + backgroundColor: 'rgb(187,40,193, 0.5)', borderWidth: 2, pointRadius: 0, } ], }, options: { - responsive: false, + responsive: true, title: { display: true, position: 'top', @@ -280,12 +332,13 @@ html mixin formatNumber(num) = Number(num).toLocaleString() - mixin sortableLinks(col, label) - div.d-inline-flex - span.sortables.mr-2.d-inline-flex.flex-column(style="font-size: 50%") + mixin sortableLinks(col, notCentered) + div.d-flex(class=(!notCentered ? "justify-content-center" : "")) + div.sortables.mr-2(style="font-size: 50%") a(href="#sort:" + col + ":asc") ▲ + br a(href="#sort:" + col + ":desc") ▼ - span.d-inline-block.text-truncate + div block mixin heroChart() @@ -304,13 +357,13 @@ html autocomplete="off" disabled ) Logarithmic - canvas.mx-auto(id="main-chart" width="800" height="450") + canvas.mx-auto(id="main-chart" width="1024" height="576") - const growthRate = '+' + (data.deathGrowthRate * 100).toFixed(2) + '%'; const population = 'pop. ' + data.population.toLocaleString(); const deathsPerMillion = Math.round(data.deathsPerMillion).toLocaleString() + '/MM'; const heroTitle = [ - 'Covid-19 Deaths: ' + data.name, + 'Covid-19: ' + data.name, `${population} | ${deathsPerMillion} | ${growthRate}` ]; script. @@ -322,6 +375,8 @@ html !{JSON.stringify(data.timeSeriesDaily.map(x => x.delta))}, !{JSON.stringify(data.rollingAverageDaily.map(x => x.delta))}, !{JSON.stringify(data.doublingDaily.map(x => x.value))}, + !{JSON.stringify(data.cases.timeSeriesDaily.map(x => x.value))}, + !{JSON.stringify(data.cases.timeSeriesDaily.map(x => x.delta))}, ); mixin dataTable(items, label, type) @@ -329,18 +384,27 @@ html div#table.table-responsive: table.table.table-sm.table-hover thead: tr - th # - th(data-col="name"): +sortableLinks("name")= label + th.text-center.font-weight-bold.geo-bg-dark(colspan=(hasPopulation ? 3 : 2)) Geography + th.text-center.font-weight-bold.cases-bg-dark(colspan="2") Cases + th.text-center.font-weight-bold.deaths-bg-dark(colspan="100") Deaths + thead.headers: tr + th.geo-bg # + th.geo-bg(data-col="name"): +sortableLinks("name", true)= label if hasPopulation - th.text-center(data-col="population"): +sortableLinks("population") Population - th.text-center(data-col="million"): +sortableLinks("million") per 1M - th.text-center(data-col="total"): +sortableLinks("total") Total - th.text-center.sorted(data-col="today"): +sortableLinks("today") Today - th.text-center(data-col="yesterday"): +sortableLinks("yesterday") Yesterday - th.text-center(data-col="last7"): +sortableLinks("last7") Last 7 - th.text-center(data-col="last30"): +sortableLinks("last30") Last 30 - th.text-center(data-col="growth"): +sortableLinks("growth") Growth - th.text-center Trend + th.geo-bg(data-col="population"): +sortableLinks("population") Population + + th.cases-bg(data-col="cases-total"): +sortableLinks("cases-total") Total + th.cases-bg(data-col="cases-today"): +sortableLinks("cases-today") Today + + if hasPopulation + th.deaths-bg(data-col="million"): +sortableLinks("million") per 1M + th.deaths-bg(data-col="total"): +sortableLinks("total") Total + th.sorted.deaths-bg(data-col="today"): +sortableLinks("today") Today + th.deaths-bg(data-col="yesterday"): +sortableLinks("yesterday") Yesterday + th.deaths-bg(data-col="last7"): +sortableLinks("last7") Last 7 + th.deaths-bg(data-col="last30"): +sortableLinks("last30") Last 30 + th.deaths-bg(data-col="growth"): +sortableLinks("growth") Growth + th.text-center.deaths-bg Trend - items.sort((a, b) => { @@ -357,13 +421,19 @@ html - const getValue = offset => (item.timeSeriesDaily[item.timeSeriesDaily.length - offset] || {}).value || 0; const getDelta = offset => (item.timeSeriesDaily[item.timeSeriesDaily.length - offset] || {}).delta || 0; + const getValueCases = offset => (item.cases.timeSeriesDaily[item.cases.timeSeriesDaily.length - offset] || {}).value || 0; + const getDeltaCases = offset => (item.cases.timeSeriesDaily[item.cases.timeSeriesDaily.length - offset] || {}).delta || 0; const today = getDelta(1); const yesterday = getDelta(2); const last7 = getValue(1) - getValue(7); const last30 = getValue(1) - getValue(30); + const casesTotal = getValueCases(1); + const casesToday = getDeltaCases(1); tr( id=("row-" + (item.safeName || '_')) data-name=(item.name || '_') + data-cases-total=casesTotal + data-cases-today=casesToday data-population=item.population data-total=item.total data-million=item.deathsPerMillion @@ -377,6 +447,9 @@ html td: +renderItemName(item) if hasPopulation td.text-right: code: +formatNumber(item.population) + td.text-right: code: +formatNumber(casesTotal) + td.text-right: code: +formatNumber(casesToday) + if hasPopulation td.text-right: code: +formatNumber(Math.round(item.deathsPerMillion)) td.text-right: code: +formatNumber(item.total) td.text-right.sorted: code: +formatNumber(today) @@ -385,7 +458,7 @@ html td.text-right: code: +formatNumber(last30) td.text-right: code= Number(item.deathGrowthRate * 100).toFixed(2) + '%' td - canvas.mx-auto(id="sparkline-" + i width="200" height="50") + canvas.mx-auto(id="sparkline-" + i width="100" height="35") script. makeSparkline( "sparkline-#{i}", @@ -393,8 +466,9 @@ html ); + div.container.mt-2 - h1.text-center Covid-19 Death Data + h1.text-center Covid-19 Data div.d-flex.justify-content-around.font-italic.small div - const generationDate = new Date().toISOString(); @@ -413,7 +487,7 @@ html script. (function() { const table = document.getElementById('table'); - const headerRow = table.querySelector('thead tr'); + const headerRow = table.querySelector('thead.headers tr'); const headers = [].slice.call(headerRow.querySelectorAll('th')); const tbody = table.querySelector('tbody'); const allRows = [].slice.call(tbody.querySelectorAll('tbody tr')); @@ -485,6 +559,8 @@ html resortTable('name'); break; case 'total': + case 'cases-total': + case 'cases-today': case 'today': case 'yesterday': case 'last7':