tmont/covid19
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added case data

Browse Source
This commit is contained in:
tmont 2020-05-02 12:27:46 -07:00
parent b86ac75ffd
commit a4afdeb8a6
2 changed files with 331 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

250
generate.js
View File

@ -62,7 +62,7 @@ const lastUpdate = new Date(lastGlobalDeathsUpdate > lastUSDeathsUpdate ?
const zeroPad = value => value < 10 ? `0${value}` : value.toString();
const toSafeName = x => x.replace(/[^A-Za-z]/g, '-').toLowerCase();
const processGlobalDeaths = async () => {
const processRecords = async () => {
const globalStart = Date.now();
let start = Date.now();
@ -81,6 +81,14 @@ const processGlobalDeaths = async () => {
const populationCountriesRaw = fs.readFileSync(populationCountriesCsv, {encoding: 'utf8'});
console.log(`read countries population CSV in ${Date.now() - start}ms`);
start = Date.now();
const casesGlobalRaw = fs.readFileSync(confirmedGlobalCsv, {encoding: 'utf8'});
console.log(`read global confirmed CSV in ${Date.now() - start}ms`);
start = Date.now();
const casesUSRaw = fs.readFileSync(confirmedUSCsv, {encoding: 'utf8'});
console.log(`read US confirmed CSV in ${Date.now() - start}ms`);
start = Date.now();
let tsGlobalRecords = parseCsv(timeSeriesGlobalRaw, {
cast: true,
@ -95,6 +103,20 @@ const processGlobalDeaths = async () => {
});
console.log(`parsed US deaths CSV in ${Date.now() - start}ms`);
start = Date.now();
let tsCasesGlobal = parseCsv(casesGlobalRaw, {
cast: true,
columns: true,
});
console.log(`parsed global cases CSV in ${Date.now() - start}ms`);
start = Date.now();
let tsCasesUS = parseCsv(casesUSRaw, {
cast: true,
columns: true,
});
console.log(`parsed US cases CSV in ${Date.now() - start}ms`);
start = Date.now();
let populationUSStateRecords = parseCsv(populationUSRaw, {
cast: true,
@ -217,31 +239,7 @@ const processGlobalDeaths = async () => {
return calcGrowthRate(record.timeSeriesDaily, record.timeSeriesDaily.length - 1, 7);
};
// state/county data is separated for the US and doesn't need to be rolled up
tsUSRecords.forEach((usRecord) => {
const newRecord = {
...usRecord,
needsRollup: false,
Long: usRecord.Long_,
'Province/State': usRecord.Province_State,
'Country/Region': usRecord.Country_Region,
};
delete newRecord.UID;
delete newRecord.iso2;
delete newRecord.iso3;
delete newRecord.code3;
delete newRecord.FIPS;
delete newRecord.Combined_Key;
delete newRecord.Long_;
delete newRecord.Province_State;
delete newRecord.Country_Region;
tsGlobalRecords.push(newRecord);
});
start = Date.now();
tsGlobalRecords.forEach((record) => {
const normalizeRecord = (record) => {
record.timeSeriesDaily = [];
record.timeSeriesMonthly = [];
const dateColumns = Object.keys(record).filter(x => /^\d+\/\d+\/\d+$/.test(x))
@ -335,6 +333,81 @@ const processGlobalDeaths = async () => {
delete record.Long;
delete record.Admin2;
delete record.Population;
};
const getRecordKey = record => `${record.country || ''}:${record.state || ''}:${record.county || ''}`;
// pre-process confirmed case data for later lookup
tsCasesUS.forEach((usRecord) => {
const newRecord = {
...usRecord,
needsRollup: false,
Long: usRecord.Long_,
'Province/State': usRecord.Province_State,
'Country/Region': usRecord.Country_Region,
}
delete newRecord.UID;
delete newRecord.iso2;
delete newRecord.iso3;
delete newRecord.code3;
delete newRecord.FIPS;
delete newRecord.Combined_Key;
delete newRecord.Long_;
delete newRecord.Province_State;
delete newRecord.Country_Region;
tsCasesGlobal.push(newRecord);
});
const confirmedCasesLookup = tsCasesGlobal.reduce((lookup, record) => {
normalizeRecord(record);
const key = getRecordKey(record);
if (lookup[key]) {
throw new Error(`key "${key}" already exists in confirmed case lookup table`);
}
lookup[key] = record;
return lookup;
}, {});
// state/county data is separated for the US and doesn't need to be rolled up
tsUSRecords.forEach((usRecord) => {
const newRecord = {
...usRecord,
needsRollup: false,
Long: usRecord.Long_,
'Province/State': usRecord.Province_State,
'Country/Region': usRecord.Country_Region,
};
delete newRecord.UID;
delete newRecord.iso2;
delete newRecord.iso3;
delete newRecord.code3;
delete newRecord.FIPS;
delete newRecord.Combined_Key;
delete newRecord.Long_;
delete newRecord.Province_State;
delete newRecord.Country_Region;
tsGlobalRecords.push(newRecord);
});
start = Date.now();
tsGlobalRecords.forEach((record) => {
normalizeRecord(record);
const recordKey = getRecordKey(record);
const confirmedCases = confirmedCasesLookup[recordKey];
if (!confirmedCases) {
throw new Error(`no cases found in lookup for key "${recordKey}"`);
}
record.cases = {
timeSeriesDaily: confirmedCases.timeSeriesDaily,
timeSeriesMonthly: confirmedCases.timeSeriesMonthly,
rollingAverageDaily: confirmedCases.rollingAverageDaily,
};
if (!record.population && !record.state && !record.county) {
const mappedPop = countryPopulationMap[record.country];
@ -376,6 +449,10 @@ const processGlobalDeaths = async () => {
timeSeriesMonthly: {},
states: [],
safeName: record.countrySafeName,
cases: {
timeSeriesDaily: {},
timeSeriesMonthly: {},
},
};
const item = perCountryTotals[record.country];
@ -404,6 +481,10 @@ const processGlobalDeaths = async () => {
deathsPerMillion: 0,
timeSeriesDaily: {},
timeSeriesMonthly: {},
cases: {
timeSeriesDaily: {},
timeSeriesMonthly: {},
},
counties: [],
};
@ -436,6 +517,24 @@ const processGlobalDeaths = async () => {
stateItem.timeSeriesMonthly[ts.key].delta += ts.delta;
});
record.cases.timeSeriesDaily.forEach((ts) => {
stateItem.cases.timeSeriesDaily[ts.key] = stateItem.cases.timeSeriesDaily[ts.key] || {
value: 0,
delta: 0,
};
stateItem.cases.timeSeriesDaily[ts.key].value += ts.value;
stateItem.cases.timeSeriesDaily[ts.key].delta += ts.delta;
});
record.cases.timeSeriesMonthly.forEach((ts) => {
stateItem.cases.timeSeriesMonthly[ts.key] = stateItem.cases.timeSeriesMonthly[ts.key] || {
value: 0,
delta: 0,
};
stateItem.cases.timeSeriesMonthly[ts.key].value += ts.value;
stateItem.cases.timeSeriesMonthly[ts.key].delta += ts.delta;
});
stateItem.counties.push(record);
} else {
item.states.push(record);
@ -463,10 +562,31 @@ const processGlobalDeaths = async () => {
item.timeSeriesMonthly[ts.key].value += ts.value;
item.timeSeriesMonthly[ts.key].delta += ts.delta;
});
record.cases.timeSeriesDaily.forEach((ts) => {
item.cases.timeSeriesDaily[ts.key] = item.cases.timeSeriesDaily[ts.key] || {
value: 0,
delta: 0,
};
item.cases.timeSeriesDaily[ts.key].value += ts.value;
item.cases.timeSeriesDaily[ts.key].delta += ts.delta;
});
record.cases.timeSeriesMonthly.forEach((ts) => {
item.cases.timeSeriesMonthly[ts.key] = item.cases.timeSeriesMonthly[ts.key] || {
value: 0,
delta: 0,
};
item.cases.timeSeriesMonthly[ts.key].value += ts.value;
item.cases.timeSeriesMonthly[ts.key].delta += ts.delta;
});
});
Object.keys(perStateTotals).forEach((stateName) => {
const item = perStateTotals[stateName];
if (!item.cases) {
throw new Error('no cases');
}
const stateItem = {
name: stateName,
safeName: item.safeName,
@ -490,11 +610,28 @@ const processGlobalDeaths = async () => {
delta: item.timeSeriesMonthly[date].delta,
};
}),
cases: {
timeSeriesDaily: Object.keys(item.cases.timeSeriesDaily).sort().map((date) => {
return {
key: date,
value: item.cases.timeSeriesDaily[date].value,
delta: item.cases.timeSeriesDaily[date].delta,
};
}),
timeSeriesMonthly: Object.keys(item.cases.timeSeriesMonthly).sort().map((date) => {
return {
key: date,
value: item.cases.timeSeriesMonthly[date].value,
delta: item.cases.timeSeriesMonthly[date].delta,
};
}),
},
};
stateItem.deathGrowthRate = getGrowthRate(stateItem);
stateItem.rollingAverageDaily = getRollingAverage(stateItem);
stateItem.doublingDaily = getDoublingTime(stateItem);
stateItem.cases.rollingAverageDaily = getRollingAverage(stateItem.cases);
// insert into states array for the country
perCountryTotals[item.country].states.push(stateItem);
@ -509,6 +646,10 @@ const processGlobalDeaths = async () => {
item.population = countryPopulationMap[countryName];
}
if (!item.cases) {
throw new Error('no cases for country');
}
const countryItem = {
name: countryName,
safeName: item.safeName,
@ -530,11 +671,28 @@ const processGlobalDeaths = async () => {
delta: item.timeSeriesMonthly[date].delta,
};
}),
cases: {
timeSeriesDaily: Object.keys(item.cases.timeSeriesDaily).sort().map((date) => {
return {
key: date,
value: item.cases.timeSeriesDaily[date].value,
delta: item.cases.timeSeriesDaily[date].delta,
};
}),
timeSeriesMonthly: Object.keys(item.cases.timeSeriesMonthly).sort().map((date) => {
return {
key: date,
value: item.cases.timeSeriesMonthly[date].value,
delta: item.cases.timeSeriesMonthly[date].delta,
};
}),
},
};
countryItem.deathGrowthRate = getGrowthRate(countryItem);
countryItem.rollingAverageDaily = getRollingAverage(countryItem);
countryItem.doublingDaily = getDoublingTime(countryItem);
countryItem.cases.rollingAverageDaily = getRollingAverage(countryItem.cases);
return countryItem;
});
@ -545,6 +703,10 @@ const processGlobalDeaths = async () => {
countries: countryArr,
timeSeriesDaily: {},
timeSeriesMonthly: {},
cases: {
timeSeriesDaily: {},
timeSeriesMonthly: {},
},
};
countryArr.forEach((countryData) => {
@ -567,6 +729,24 @@ const processGlobalDeaths = async () => {
worldData.timeSeriesMonthly[ts.key].value += ts.value;
worldData.timeSeriesMonthly[ts.key].delta += ts.delta;
});
countryData.cases.timeSeriesDaily.forEach((ts) => {
worldData.cases.timeSeriesDaily[ts.key] = worldData.cases.timeSeriesDaily[ts.key] || {
value: 0,
delta: 0,
};
worldData.cases.timeSeriesDaily[ts.key].value += ts.value;
worldData.cases.timeSeriesDaily[ts.key].delta += ts.delta;
});
countryData.cases.timeSeriesMonthly.forEach((ts) => {
worldData.cases.timeSeriesMonthly[ts.key] = worldData.cases.timeSeriesMonthly[ts.key] || {
value: 0,
delta: 0,
};
worldData.cases.timeSeriesMonthly[ts.key].value += ts.value;
worldData.cases.timeSeriesMonthly[ts.key].delta += ts.delta;
});
});
worldData.timeSeriesDaily = Object.keys(worldData.timeSeriesDaily).sort().map((date) => {
@ -583,10 +763,25 @@ const processGlobalDeaths = async () => {
delta: worldData.timeSeriesMonthly[date].delta,
};
});
worldData.cases.timeSeriesDaily = Object.keys(worldData.cases.timeSeriesDaily).sort().map((date) => {
return {
key: date,
value: worldData.cases.timeSeriesDaily[date].value,
delta: worldData.cases.timeSeriesDaily[date].delta,
};
});
worldData.cases.timeSeriesMonthly = Object.keys(worldData.cases.timeSeriesMonthly).sort().map((date) => {
return {
key: date,
value: worldData.cases.timeSeriesMonthly[date].value,
delta: worldData.cases.timeSeriesMonthly[date].delta,
};
});
worldData.deathGrowthRate = getGrowthRate(worldData);
worldData.rollingAverageDaily = getRollingAverage(worldData);
worldData.doublingDaily = getDoublingTime(worldData);
worldData.cases.rollingAverageDaily = getRollingAverage(worldData.cases);
worldData.population = 7781841000;
worldData.deathsPerMillion = worldData.total / worldData.population * 1000000;
@ -604,7 +799,6 @@ const processGlobalDeaths = async () => {
const targetFile = path.join(publicDir, 'index.html');
fs.writeFileSync(targetFile, worldHtml);
console.log(`wrote to ${targetFile} in ${Date.now() - start}ms`);
// fs.writeFileSync(path.join(publicDir, 'countries.json'), JSON.stringify(countryArr, null, ' '));
const singleCountryTmpl = path.join(templatesDir, 'country.pug');
const singleStateTmpl = path.join(templatesDir, 'state.pug');
@ -645,7 +839,7 @@ const processGlobalDeaths = async () => {
console.log(`finished in ${((Date.now() - globalStart) / 1000).toFixed(2)}s`);
};
processGlobalDeaths()
processRecords()
.then(() => {
console.log('all done');
})

142
tmpl/master.pug
View File

@ -11,7 +11,28 @@ html
vertical-align: middle !important;
}
th.sorted, td.sorted {
background-color: #e0eefd;
background-color: #e0eaf7;
}
.geo-bg-dark {
background-color: #8e8e8e;
color: white;
}
.cases-bg-dark {
background-color: #aaa55e;
color: white;
}
.deaths-bg-dark {
background-color: #a65353;
color: white;
}
.geo-bg {
background-color: #eeeeee;
}
.cases-bg {
background-color: #f9f6d5;
}
.deaths-bg {
background-color: #eac8c8;
}
.table-sm {
font-size: 80%;
@ -170,9 +191,19 @@ html
});
}
function makeHeroChart(id, title, labels, totalDeaths, newDeaths, rollingAverage, doubling) {
function makeHeroChart(
id,
title,
labels,
totalDeaths,
newDeaths,
rollingAverage,
doubling,
totalCases,
newCases,
) {
const canvas = document.getElementById(id);
charts.heroMaxValue = totalDeaths[totalDeaths.length - 1];
charts.heroMaxValue = totalCases.reduce((max, value) => Math.max(max, value), 0);
const firstNonZeroDeathIndex = totalDeaths.findIndex(value => value > 0);
const start = Math.max(0, firstNonZeroDeathIndex - 2);
@ -182,6 +213,8 @@ html
const newData = newDeaths.slice(start, end);
const rollingData = rollingAverage.slice(start, end);
const doublingData = doubling.slice(start, end);
const totalCaseData = totalCases.slice(start, end);
const newCaseData = newCases.slice(start, end);
const months = [ 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec' ];
@ -195,40 +228,59 @@ html
labels: realLabels,
datasets: [
{
label: 'Cumulative',
data: totalData,
fill: '1',
borderColor: 'rgb(196, 64, 64)',
label: 'Cases',
data: totalCaseData,
fill: '2',
borderColor: 'rgb(161,150,20)',
backgroundColor: 'rgba(161,150,20, 0.25)',
borderWidth: 1,
backgroundColor: 'rgba(196, 128, 128, 0.25)',
},
{
label: 'New (rolling)',
label: 'New Cases',
data: newCaseData,
fill: false,
borderColor: 'rgba(206,108,46,0.5)',
backgroundColor: 'rgba(206,108,46,0.5)',
borderWidth: 1,
pointRadius: 0,
},
{
label: 'Deaths',
data: totalData,
fill: '3',
borderColor: 'rgb(196, 64, 64)',
backgroundColor: 'rgba(196, 64, 64, 0.25)',
borderWidth: 1,
},
{
label: 'New Deaths (rolling)',
data: rollingData,
fill: 'origin',
borderColor: 'rgb(20,24,59)',
borderWidth: 1,
backgroundColor: 'rgba(96, 96, 164, 0.25)',
borderWidth: 1,
},
{
label: 'New',
label: 'New Deaths',
data: newData,
fill: false,
borderColor: 'rgb(96, 96, 96, 0.25)',
backgroundColor: 'rgb(96, 96, 96, 0.25)',
borderWidth: 1,
},
{
label: 'Days to 2x',
label: 'Days to 2x deaths',
data: doublingData,
fill: false,
borderColor: 'rgb(187,40,193, 0.5)',
backgroundColor: 'rgb(187,40,193, 0.5)',
borderWidth: 2,
pointRadius: 0,
}
],
},
options: {
responsive: false,
responsive: true,
title: {
display: true,
position: 'top',
@ -280,12 +332,13 @@ html
mixin formatNumber(num)
= Number(num).toLocaleString()
mixin sortableLinks(col, label)
div.d-inline-flex
span.sortables.mr-2.d-inline-flex.flex-column(style="font-size: 50%")
mixin sortableLinks(col, notCentered)
div.d-flex(class=(!notCentered ? "justify-content-center" : ""))
div.sortables.mr-2(style="font-size: 50%")
a(href="#sort:" + col + ":asc") ▲
br
a(href="#sort:" + col + ":desc") ▼
span.d-inline-block.text-truncate
div
block
mixin heroChart()
@ -304,13 +357,13 @@ html
autocomplete="off"
disabled
) Logarithmic
canvas.mx-auto(id="main-chart" width="800" height="450")
canvas.mx-auto(id="main-chart" width="1024" height="576")
-
const growthRate = '+' + (data.deathGrowthRate * 100).toFixed(2) + '%';
const population = 'pop. ' + data.population.toLocaleString();
const deathsPerMillion = Math.round(data.deathsPerMillion).toLocaleString() + '/MM';
const heroTitle = [
'Covid-19 Deaths: ' + data.name,
'Covid-19: ' + data.name,
`${population} | ${deathsPerMillion} | ${growthRate}`
];
script.
@ -322,6 +375,8 @@ html
!{JSON.stringify(data.timeSeriesDaily.map(x => x.delta))},
!{JSON.stringify(data.rollingAverageDaily.map(x => x.delta))},
!{JSON.stringify(data.doublingDaily.map(x => x.value))},
!{JSON.stringify(data.cases.timeSeriesDaily.map(x => x.value))},
!{JSON.stringify(data.cases.timeSeriesDaily.map(x => x.delta))},
);
mixin dataTable(items, label, type)
@ -329,18 +384,27 @@ html
div#table.table-responsive: table.table.table-sm.table-hover
thead: tr
th #
th(data-col="name"): +sortableLinks("name")= label
th.text-center.font-weight-bold.geo-bg-dark(colspan=(hasPopulation ? 3 : 2)) Geography
th.text-center.font-weight-bold.cases-bg-dark(colspan="2") Cases
th.text-center.font-weight-bold.deaths-bg-dark(colspan="100") Deaths
thead.headers: tr
th.geo-bg #
th.geo-bg(data-col="name"): +sortableLinks("name", true)= label
if hasPopulation
th.text-center(data-col="population"): +sortableLinks("population") Population
th.text-center(data-col="million"): +sortableLinks("million") per 1M
th.text-center(data-col="total"): +sortableLinks("total") Total
th.text-center.sorted(data-col="today"): +sortableLinks("today") Today
th.text-center(data-col="yesterday"): +sortableLinks("yesterday") Yesterday
th.text-center(data-col="last7"): +sortableLinks("last7") Last 7
th.text-center(data-col="last30"): +sortableLinks("last30") Last 30
th.text-center(data-col="growth"): +sortableLinks("growth") Growth
th.text-center Trend
th.geo-bg(data-col="population"): +sortableLinks("population") Population
th.cases-bg(data-col="cases-total"): +sortableLinks("cases-total") Total
th.cases-bg(data-col="cases-today"): +sortableLinks("cases-today") Today
if hasPopulation
th.deaths-bg(data-col="million"): +sortableLinks("million") per 1M
th.deaths-bg(data-col="total"): +sortableLinks("total") Total
th.sorted.deaths-bg(data-col="today"): +sortableLinks("today") Today
th.deaths-bg(data-col="yesterday"): +sortableLinks("yesterday") Yesterday
th.deaths-bg(data-col="last7"): +sortableLinks("last7") Last 7
th.deaths-bg(data-col="last30"): +sortableLinks("last30") Last 30
th.deaths-bg(data-col="growth"): +sortableLinks("growth") Growth
th.text-center.deaths-bg Trend
-
items.sort((a, b) => {
@ -357,13 +421,19 @@ html
-
const getValue = offset => (item.timeSeriesDaily[item.timeSeriesDaily.length - offset] || {}).value || 0;
const getDelta = offset => (item.timeSeriesDaily[item.timeSeriesDaily.length - offset] || {}).delta || 0;
const getValueCases = offset => (item.cases.timeSeriesDaily[item.cases.timeSeriesDaily.length - offset] || {}).value || 0;
const getDeltaCases = offset => (item.cases.timeSeriesDaily[item.cases.timeSeriesDaily.length - offset] || {}).delta || 0;
const today = getDelta(1);
const yesterday = getDelta(2);
const last7 = getValue(1) - getValue(7);
const last30 = getValue(1) - getValue(30);
const casesTotal = getValueCases(1);
const casesToday = getDeltaCases(1);
tr(
id=("row-" + (item.safeName || '_'))
data-name=(item.name || '_')
data-cases-total=casesTotal
data-cases-today=casesToday
data-population=item.population
data-total=item.total
data-million=item.deathsPerMillion
@ -377,6 +447,9 @@ html
td: +renderItemName(item)
if hasPopulation
td.text-right: code: +formatNumber(item.population)
td.text-right: code: +formatNumber(casesTotal)
td.text-right: code: +formatNumber(casesToday)
if hasPopulation
td.text-right: code: +formatNumber(Math.round(item.deathsPerMillion))
td.text-right: code: +formatNumber(item.total)
td.text-right.sorted: code: +formatNumber(today)
@ -385,7 +458,7 @@ html
td.text-right: code: +formatNumber(last30)
td.text-right: code= Number(item.deathGrowthRate * 100).toFixed(2) + '%'
td
canvas.mx-auto(id="sparkline-" + i width="200" height="50")
canvas.mx-auto(id="sparkline-" + i width="100" height="35")
script.
makeSparkline(
"sparkline-#{i}",
@ -393,8 +466,9 @@ html
);
div.container.mt-2
h1.text-center Covid-19 Death Data
h1.text-center Covid-19 Data
div.d-flex.justify-content-around.font-italic.small
div
- const generationDate = new Date().toISOString();
@ -413,7 +487,7 @@ html
script.
(function() {
const table = document.getElementById('table');
const headerRow = table.querySelector('thead tr');
const headerRow = table.querySelector('thead.headers tr');
const headers = [].slice.call(headerRow.querySelectorAll('th'));
const tbody = table.querySelector('tbody');
const allRows = [].slice.call(tbody.querySelectorAll('tbody tr'));
@ -485,6 +559,8 @@ html
resortTable('name');
break;
case 'total':
case 'cases-total':
case 'cases-today':
case 'today':
case 'yesterday':
case 'last7':