DWR assimilation: Argentine cold surge of August 1902 video¶
This case-study shows the effect of using observations from the Argentine Daily Weather Reports, newly digitised through the Copernicus C3S Data Rescue Service, to improve the reconstruction of the 1902 cold-surge which badly damaged the South American Coffee plantations.
On the left, a Spaghetti-contour plot of 20CRv3 MSLP. Centre and right, Scatter-contour plot comparing the same field, after assimilating all the Argentine DWR station pressure observations within 12 hours of that date. The centre panel shows the 20CRv3 ensemble after assimilating all new station observations (red dots). The right panel compares the two ensembles with the new observations: Black lines show the observed pressures, blue dots the original 20CRv3 ensemble at the station locations, and red dots the 20CR ensemble after assimilating all the observations except the observation at that location.
Code to make the figure¶
Collect the reanalysis data (prmsl ensemble and observations from 20CRv3 for February 1903):
#!/usr/bin/env python
import IRData.twcr as twcr
import datetime
dte=datetime.datetime(1902,8,1)
for version in (['4.5.1']):
twcr.fetch('prmsl',dte,version=version)
twcr.fetch('air.2m',dte,version=version)
twcr.fetch('tmp',dte,level=925,version=version)
twcr.fetch_observations(dte,version=version)
Script to calculate a leave-one-out assimilation for one station:
#!/usr/bin/env python
# Assimilate all but one station for a given date
# and store the resulting field.
import os
import math
import datetime
import numpy
import pandas
import pickle
import iris
import iris.analysis
import IRData.twcr as twcr
import SEF
import glob
import DIYA
import sklearn
RANDOM_SEED = 5
# Try and get round thread errors on spice
import dask
dask.config.set(scheduler='single-threaded')
obs_error=5 # Pa
model=sklearn.linear_model.Lasso(normalize=True)
skip_stations=['DWR_Roca_Rio_N.','DWR_Sierra_Grnde',
'DWR_Villa_Maria','DWR_Santo_Tome','DWR_Corrientes-C',
'DWR_Santa_Fa-Cp','DWR_San_Lorenzo','DWR_Carcarana',
'DWR_Estc_Pereyra','DWR_Puerto_Mili.']
opdir="%s/images/ADWR_jacknife" % os.getenv('SCRATCH')
if not os.path.isdir(opdir):
os.makedirs(opdir)
# Assimilate observations within this range of the field
hours_before=12
hours_after=12
# Get the datetime, and station to omit, from commandline arguments
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--year", help="Year",
type=int,required=True)
parser.add_argument("--month", help="Integer month",
type=int,required=True)
parser.add_argument("--day", help="Day of month",
type=int,required=True)
parser.add_argument("--hour", help="Time of day (0,3,6,9,12,18,or 21)",
type=int,required=True)
parser.add_argument("--omit", help="Name of station to be left out",
default=None,
type=str,required=False)
args = parser.parse_args()
dte=datetime.datetime(args.year,args.month,args.day,
int(args.hour),int(args.hour%1*60))
# 20CRv3 data
prmsl=twcr.load('prmsl',dte,version='4.5.1')
# Get the DWR observations close to the selected time
ob_start=dte-datetime.timedelta(hours=hours_before)
ob_end=dte+datetime.timedelta(hours=hours_after)
adf=glob.glob('../../../../../../station-data/ToDo/sef/Argentinian_DWR/1902/DWR_*_MSLP.tsv')
obs={'name': [], 'latitude': [], 'longitude': [], 'value' : [], 'dtm' : []}
for file in adf:
stobs=SEF.read_file(file)
df=stobs['Data']
# Add the datetime
df=df.assign(Hour=(df['HHMM']/100).astype(int))
df=df.assign(Minute=(df['HHMM']%100).astype(int))
df=df.assign(dtm=pandas.to_datetime(df[['Year','Month','Day','Hour','Minute']]))
# Select the subset close to the selected time
mslp=df[(df.dtm>=ob_start) & (df.dtm<ob_end)]
if mslp.empty: continue
if mslp['Value'].values[0]==mslp['Value'].values[0]:
obs['name'].append(stobs['ID'])
obs['latitude'].append(stobs['Lat'])
obs['longitude'].append(stobs['Lon'])
obs['value'].append(mslp['Value'].values[0])
obs['dtm'].append(mslp['dtm'].values[0])
obs=pandas.DataFrame(obs)
# Throw out the suspected duds
obs=obs[~obs['name'].isin(skip_stations)]
# Throw out the specified station
if args.omit is not None:
obs=obs[~obs['name'].isin([args.omit])]
# Convert to normalised value same as reanalysis
obs.value=obs.value*100
# Update mslp by assimilating all obs.
prmsl2=DIYA.constrain_cube(prmsl,
lambda dte: twcr.load('prmsl',dte,version='4.5.1'),
obs=obs,
obs_error=obs_error,
random_state=RANDOM_SEED,
model=model,
lat_range=(-80,10),
lon_range=(250,350))
dumpfile="%s/%04d%02d%02d%02d.pkl" % (opdir,args.year,args.month,
args.day,args.hour)
if args.omit is not None:
dumpfile="%s/%04d%02d%02d%02d_%s.pkl" % (opdir,args.year,args.month,
args.day,args.hour,args.omit)
pickle.dump( prmsl2, open( dumpfile, "wb" ) )
Calculate the leave-one-out assimilations for all the stations. (This script produces a list of calculations which can be done in parallel):
#!/usr/bin/env python
# Do all the jacknife assimilations for a range of times
# Just generates the commands to run - run with gnu parallel or similar.
import os
import sys
import time
import subprocess
import datetime
import glob
import SEF
import pandas
import numpy
from collections import OrderedDict
# Assimilate observations within this range of the field
hours_before=12
hours_after=12
# Where to put the output files
opdir="%s/images/ADWR_jacknife" % os.getenv('SCRATCH')
if not os.path.isdir(opdir):
os.makedirs(opdir)
# Function to check if the job is already done for this time+omitted station
def is_done(dte,omit):
opfile="%s/%04d%02d%02d%02d.pkl" % (opdir,dte.year,dte.month,
dte.day,dte.hour)
if omit is not None:
opfile="%s/%04d%02d%02d%02d_%s.pkl" % (opdir,dte.year,dte.month,
dte.day,dte.hour,omit)
if os.path.isfile(opfile):
return True
return False
f=open("run.txt","w+")
start_day=datetime.datetime(1902, 8, 10, 18)
end_day =datetime.datetime(1902, 8, 25, 18)
dte=start_day
while dte<=end_day:
# What stations do we need to do at this point in time?
# Load all the Argentine DWR pressures at this time
adf=glob.glob('../../../../../../station-data/ToDo/sef/Argentinian_DWR/1902/DWR_*_MSLP.tsv')
obs={'Name': [], 'Latitude': [], 'Longitude': [], 'mslp' : []}
for file in adf:
stobs=SEF.read_file(file)
df=stobs['Data']
#hhmm=int("%2d%02d" % (dte.hour,17))
mslp=df.loc[(df['Year'] == dte.year) &
(df['Month'] == dte.month) &
(df['Day'] == dte.day)]
if mslp.empty: continue
if mslp['Value'].values[0]==mslp['Value'].values[0]:
obs['Name'].append(stobs['ID'])
obs['Latitude'].append(stobs['Lat'])
obs['Longitude'].append(stobs['Lon'])
obs['mslp'].append(mslp['Value'].values[0])
obs=pandas.DataFrame(obs)
stations=obs.Name.values.tolist()
stations=list(OrderedDict.fromkeys(obs.Name.values))
stations.append(None) # Also do case with all stations
for station in stations:
if is_done(dte,station): continue
cmd="./leave-one-out.py --year=%d --month=%d --day=%d --hour=%d" % (
dte.year,dte.month,dte.day,dte.hour)
if station is not None:
cmd="%s --omit=%s" % (cmd,station)
cmd="%s\n" % cmd
f.write(cmd)
dte=dte+datetime.timedelta(hours=3)
f.close()
Plot a single frame:
#!/usr/bin/env python
# Assimilation of many stations.
# Plot the precalculated leave-one-out assimilation results
import os
import math
import datetime
import numpy
import pandas
import pickle
from collections import OrderedDict
import iris
import iris.analysis
import matplotlib
from matplotlib.backends.backend_agg import \
FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
from matplotlib.patches import Circle
import cartopy
import cartopy.crs as ccrs
import Meteorographica as mg
import IRData.twcr as twcr
import SEF
import glob
import DIYA
import sklearn
RANDOM_SEED = 5
import DWR
# Try and get round thread errors on spice
import dask
dask.config.set(scheduler='single-threaded')
obs_error=5 # Pa
model=sklearn.linear_model.Lasso(normalize=True)
# Colocated stations
skip_stations=['DWR_Roca_Rio_N.','DWR_Puerto_Mili.','DWR_Corrientes-C',
'DWR_Estc_Pereyra','DWR_San_Lorenzo','DWR_Carcarana',
'DWR_Santo_Tome']
opdir="%s/images/ADWR_jacknife" % os.getenv('SCRATCH')
if not os.path.isdir(opdir):
os.makedirs(opdir)
# Assimilate observations within this range of the field
hours_before=12
hours_after=12
# Get the DWR observations in the whole range of time
ob_start=datetime.datetime(1902,8,5,0)
ob_end=datetime.datetime(1902,8,31,23)
adf=glob.glob('../../../../../../station-data/ToDo/sef/Argentinian_DWR/1902/DWR_*_MSLP.tsv')
obs={'name': [], 'latitude': [], 'longitude': [], 'value' : [], 'dtm' : []}
for file in adf:
stobs=SEF.read_file(file)
df=stobs['Data']
# Add the datetime
df=df.assign(Hour=(df['HHMM']/100).astype(int))
df=df.assign(Minute=(df['HHMM']%100).astype(int))
df=df.assign(dtm=pandas.to_datetime(df[['Year','Month','Day','Hour','Minute']]))
# Select the subset close to the selected time
mslp=df[df.Value.notnull() & (df.dtm>=ob_start) & (df.dtm<ob_end)]
if mslp.empty: continue
obs['name'].extend([stobs['ID']]*len(mslp))
obs['latitude'].extend([stobs['Lat']]*len(mslp))
obs['longitude'].extend([stobs['Lon']]*len(mslp))
obs['value'].extend(mslp['Value'].values)
obs['dtm'].extend(mslp['dtm'].values)
full_obs=pandas.DataFrame(obs)
full_obs=full_obs[~full_obs['name'].isin(skip_stations)]
full_obs=full_obs.sort_values(by='latitude',ascending=False)
stations=list(OrderedDict.fromkeys(full_obs.name.values))
# Where are the precalculated fields?
ipdir="%s/images/ADWR_jacknife" % os.getenv('SCRATCH')
# Where to put the plots?
opdir="%s/images/ADWR_jacknife_png" % os.getenv('SCRATCH')
if not os.path.isdir(opdir):
os.makedirs(opdir)
# Date to show
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("--year", help="Year",
type=int,required=True)
parser.add_argument("--month", help="Integer month",
type=int,required=True)
parser.add_argument("--day", help="Day of month",
type=int,required=True)
parser.add_argument("--hour", help="Hour of day (0 to 23.99)",
type=float,required=True)
args = parser.parse_args()
dte=datetime.datetime(args.year,args.month,args.day,
int(args.hour),int(args.hour%1*60))
# load the assimilated field with interpolation where needed
def load_interpolated(dtel,omit):
if dtel.hour%3==0 and dtel.minute==0:
if omit is not None:
dumpfile="%s/%04d%02d%02d%02d_%s.pkl" % (ipdir,dtel.year,dtel.month,
dtel.day,dtel.hour,omit)
if not os.path.isfile(dumpfile):
dumpfile="%s/%04d%02d%02d%02d.pkl" % (ipdir,dtel.year,dtel.month,
dtel.day,dtel.hour)
else:
dumpfile="%s/%04d%02d%02d%02d.pkl" % (ipdir,dtel.year,dtel.month,
dtel.day,dtel.hour)
result=pickle.load( open( dumpfile, 'rb'))
return result
else:
dte_prev=datetime.datetime(dtel.year,dtel.month,dtel.day,dtel.hour-dtel.hour%3,0)
dte_next=dte_prev+datetime.timedelta(hours=3)
r1=load_interpolated(dte_prev,omit)
r2=load_interpolated(dte_next,omit)
r1.attributes=r2.attributes
r1=iris.cube.CubeList((r1,r2)).merge_cube()
result=r1.interpolate([('time',dtel)],iris.analysis.Linear())
return result
# Landscape page
aspect=16/9.0 # HD video size
fig=Figure(figsize=(10.8*aspect,10.8), # Width, Height (inches)
dpi=100,
facecolor=(0.88,0.88,0.88,1),
edgecolor=None,
linewidth=0.0,
frameon=False,
subplotpars=None,
tight_layout=None)
canvas=FigureCanvas(fig)
font = {'family' : 'sans-serif',
'sans-serif' : 'Arial',
'weight' : 'normal',
'size' : 12}
matplotlib.rc('font', **font)
# South America centred projection
projection=ccrs.RotatedPole(pole_longitude=120, pole_latitude=125)
scale=25
extent=[scale*-1*aspect/3.0,scale*aspect/3.0,scale*-1,scale]
# On the left - spaghetti-contour plot of original 20CRv3
ax_left=fig.add_axes([0.005,0.01,0.323,0.98],projection=projection)
ax_left.set_axis_off()
ax_left.set_extent(extent, crs=projection)
ax_left.background_patch.set_facecolor((0.88,0.88,0.88,1))
mg.background.add_grid(ax_left)
land_img_left=ax_left.background_img(name='GreyT', resolution='low')
# 20CRv3 data
prmsl=twcr.load('prmsl',dte,version='4.5.1')
# 20CRv3 data
prmsl=twcr.load('prmsl',dte,version='4.5.1')
obs_t=twcr.load_observations_fortime(dte,version='4.5.1')
# Plot the observations
mg.observations.plot(ax_left,obs_t,radius=0.15)
# PRMSL spaghetti plot
mg.pressure.plot(ax_left,prmsl,scale=0.01,type='spaghetti',
resolution=0.25,
levels=numpy.arange(875,1050,10),
colors='blue',
label=False,
linewidths=0.1)
# Add the ensemble mean - with labels
prmsl_m=prmsl.collapsed('member', iris.analysis.MEAN)
prmsl_s=prmsl.collapsed('member', iris.analysis.STD_DEV)
prmsl_m.data[numpy.where(prmsl_s.data>300)]=numpy.nan
mg.pressure.plot(ax_left,prmsl_m,scale=0.01,
resolution=0.25,
levels=numpy.arange(875,1050,10),
colors='black',
label=True,
linewidths=2)
mg.utils.plot_label(ax_left,
'20CRv3',
fontsize=12,
facecolor=fig.get_facecolor(),
x_fraction=0.04,
horizontalalignment='left')
mg.utils.plot_label(ax_left,
'%04d-%02d-%02d:%02d' % (dte.year,dte.month,dte.day,dte.hour),
fontsize=12,
facecolor=fig.get_facecolor(),
x_fraction=0.96,
horizontalalignment='right')
# In the centre - spaghetti-contour plot of 20CRv3 with DWR assimilated
ax_centre=fig.add_axes([0.335,0.01,0.323,0.98],projection=projection)
ax_centre.set_axis_off()
ax_centre.set_extent(extent, crs=projection)
ax_centre.background_patch.set_facecolor((0.88,0.88,0.88,1))
mg.background.add_grid(ax_centre)
land_img_centre=ax_centre.background_img(name='GreyT', resolution='low')
# Load the all-stations-assimilated mslp
prmsl2=load_interpolated(dte,None)
# Plot the obs used in 20CRv3
mg.observations.plot(ax_centre,obs_t,radius=0.15)
# Get the obs assimilated into this field
ob_start=dte-datetime.timedelta(hours=hours_before)
ob_end=dte+datetime.timedelta(hours=hours_after)
adf=glob.glob('../../../../../../station-data/ToDo/sef/Argentinian_DWR/1902/DWR_*_MSLP.tsv')
obs={'name': [], 'latitude': [], 'longitude': [], 'value' : [], 'dtm' : []}
for file in adf:
stobs=SEF.read_file(file)
df=stobs['Data']
# Add the datetime
df=df.assign(Hour=(df['HHMM']/100).astype(int))
df=df.assign(Minute=(df['HHMM']%100).astype(int))
df=df.assign(dtm=pandas.to_datetime(df[['Year','Month','Day','Hour','Minute']]))
# Select the subset close to the selected time
mslp=df[(df.dtm>=ob_start) & (df.dtm<ob_end)]
if mslp.empty: continue
if mslp['Value'].values[0]==mslp['Value'].values[0]:
obs['name'].append(stobs['ID'])
obs['latitude'].append(stobs['Lat'])
obs['longitude'].append(stobs['Lon'])
obs['value'].append(mslp['Value'].values[0])
obs['dtm'].append(mslp['dtm'].values[0])
obs=pandas.DataFrame(obs)
obs=obs[~obs['name'].isin(skip_stations)]
# Plot the obs assimilated
mg.observations.plot(ax_centre,obs,
radius=0.15,facecolor='red',
lat_label='latitude',
lon_label='longitude')
# Mark the stations available for the period, but with no obs close
# enough in time to be assimilated into this field.
missing_obs=full_obs[~full_obs['name'].isin(obs.name.values)]
if not missing_obs.empty:
mg.observations.plot(ax_centre,missing_obs,
radius=0.15,facecolor='black',
lat_label='latitude',
lon_label='longitude')
# PRMSL spaghetti plot
mg.pressure.plot(ax_centre,prmsl2,scale=0.01,type='spaghetti',
resolution=0.25,
levels=numpy.arange(875,1050,10),
colors='blue',
label=False,
linewidths=0.1)
# Add the ensemble mean - with labels
prmsl_m=prmsl2.collapsed('member', iris.analysis.MEAN)
prmsl_s=prmsl2.collapsed('member', iris.analysis.STD_DEV)
prmsl_m.data[numpy.where(prmsl_s.data>300)]=numpy.nan
mg.pressure.plot(ax_centre,prmsl_m,scale=0.01,
resolution=0.25,
levels=numpy.arange(875,1050,10),
colors='black',
label=True,
linewidths=2)
mg.utils.plot_label(ax_centre,
'+DWR obs',
fontsize=12,
facecolor=fig.get_facecolor(),
x_fraction=0.04,
horizontalalignment='left')
ax_right=fig.add_axes([0.74,0.05,0.255,0.94])
# x-axis
xrange=[970,1045]
ax_right.set_xlim(xrange)
ax_right.set_xlabel('')
# y-axis
ax_right.set_ylim([1,len(stations)+1])
y_locations=[x+0.5 for x in range(1,len(stations)+1)]
ax_right.yaxis.set_major_locator(
matplotlib.ticker.FixedLocator(y_locations))
ax_right.yaxis.set_major_formatter(
matplotlib.ticker.FixedFormatter(
[s[4:] for s in stations]))
# Custom grid spacing
for y in range(0,len(stations)):
ax_right.add_line(matplotlib.lines.Line2D(
xdata=xrange,
ydata=(y+1,y+1),
linestyle='solid',
linewidth=0.2,
color=(0.5,0.5,0.5,1),
zorder=0))
latlon={}
for station in stations:
latlon[station]=DWR.get_station_location(full_obs,station)
# Plot the station pressures
for y in range(0,len(stations)):
station=stations[y]
try:
(mslp,gap)=DWR.at_station_and_time_with_distance(full_obs,station,dte)
except Exception: continue
if mslp is None: continue
alpha=max(0,1-abs(gap)/3600.0)
ax_right.add_line(matplotlib.lines.Line2D(
xdata=(mslp,mslp), ydata=(y+1.1,y+1.9),
linestyle='solid',
linewidth=3,
color=(0,0,0,alpha),
zorder=1))
# for each station, plot the reanalysis ensemble at that station
interpolator = iris.analysis.Linear().interpolator(prmsl,
['latitude', 'longitude'])
for y in range(len(stations)):
station=stations[y]
ensemble=interpolator([latlon[station]['latitude'],
latlon[station]['longitude']])
ax_right.scatter(ensemble.data/100.0,
numpy.linspace(y+1.5,y+1.9,
num=len(ensemble.data)),
20,
'blue', # Color
marker='.',
edgecolors='face',
linewidths=0.0,
alpha=0.5,
zorder=0.5)
# For each station, assimilate all but that station, and plot the resulting ensemble
for y in range(len(stations)):
station=stations[y]
prmsl2=load_interpolated(dte,station)
interpolator = iris.analysis.Linear().interpolator(prmsl2,
['latitude', 'longitude'])
ensemble=interpolator([latlon[station]['latitude'],
latlon[station]['longitude']])
ax_right.scatter(ensemble.data/100.0,
numpy.linspace(y+1.1,y+1.5,
num=len(ensemble.data)),
20,
'red', # Color
marker='.',
edgecolors='face',
linewidths=0.0,
alpha=0.5,
zorder=0.5)
# Join each station name to its location on the map
# Need another axes, filling the whole fig
ax_full=fig.add_axes([0,0,1,1])
ax_full.patch.set_alpha(0.0) # Transparent background
def pos_left(idx):
station=stations[idx]
ll=latlon[station]
rp=ax_centre.projection.transform_points(ccrs.PlateCarree(),
numpy.asarray(ll['longitude']),
numpy.asarray(ll['latitude']))
new_lon=rp[:,0]
new_lat=rp[:,1]
result={}
result['x']=0.335+0.323*(new_lon-(scale*-1)*aspect/3.0)/(scale*2*aspect/3.0)
result['y']=0.01+0.98*(new_lat-(scale*-1))/(scale*2)
return result
# Label location of a station in ax_full coordinates
def pos_right(idx):
result={}
result['x']=0.668
result['y']=0.05+(0.94/len(stations))*(idx+0.5)
return result
for i in range(len(stations)):
p_left=pos_left(i)
if p_left['x']<0.335 or p_left['x']>(0.335+0.323): continue
if p_left['y']<0.005 or p_left['y']>(0.005+0.94): continue
p_right=pos_right(i)
ax_full.add_patch(Circle((p_right['x'],
p_right['y']),
radius=0.001,
facecolor=(1,0,0,1),
edgecolor=(0,0,0,1),
alpha=1,
zorder=1))
ax_full.add_line(matplotlib.lines.Line2D(
xdata=(p_left['x'],p_right['x']),
ydata=(p_left['y'],p_right['y']),
linestyle='solid',
linewidth=0.2,
color=(1,0,0,1.0),
zorder=1))
# Output as png
fig.savefig('%s/Leave_one_out_%04d%02d%02d%02d%02d.png' %
(opdir,dte.year,dte.month,dte.day,dte.hour,dte.minute))
To make the video, it is necessary to run the script above hundreds of times - giving an image for every 7-minute period. (This script produces a list of calculations which can be done in parallel)
#!/usr/bin/env python
# Make all the individual frames for a movie
import os
import sys
import subprocess
import datetime
# Where to put the output files
opdir="%s/images/ADWR_jacknife_png" % os.getenv('SCRATCH')
if not os.path.isdir(opdir):
os.makedirs(opdir)
# Function to check if the job is already done for this timepoint
def is_done(dte):
op_file_name=('%s/Leave_one_out_%04d%02d%02d%02d%02d.png' %
(opdir,dte.year,dte.month,dte.day,dte.hour,dte.minute))
if os.path.isfile(op_file_name):
return True
return False
f=open("run.txt","w+")
start_day=datetime.datetime(1902, 8, 10, 18)
end_day =datetime.datetime(1902, 8, 25, 18)
dte=start_day
while dte<=end_day:
for minutes in (0,7,15,22):
if is_done(dte+datetime.timedelta(minutes=minutes)):
continue
cmd="./plot_leave-one-out.py --year=%d --month=%d --day=%d --hour=%f \n" % (
dte.year,dte.month,dte.day,
dte.hour+(dte.minute+minutes)/60.0)
f.write(cmd)
dte=dte+datetime.timedelta(minutes=30)
f.close()
To turn the thousands of images into a movie, use ffmpeg
ffmpeg -r 24 -pattern_type glob -i ADWR_jacknife_png/\*.png \
-c:v libx264 -threads 16 -preset slow -tune animation \
-profile:v high -level 4.2 -pix_fmt yuv420p -crf 25 \
-c:a copy ADWR_assimilate.mp4