geoips.interfaces.class_based package#

Submodules#

geoips.interfaces.class_based.algorithms module#

Algorithms interface class.

class geoips.interfaces.class_based.algorithms.AlgorithmsInterface[source]#

Bases: BaseClassInterface

Base interface class for algorithms plugins.

name = 'algorithms'#
plugin_class#

alias of BaseAlgorithmPlugin

required_args: dict[str, list[str]] = {'channel_combination': ['arrays'], 'list_numpy_to_numpy': ['arrays'], 'rgb': ['arrays'], 'scalar_to_scalar': [], 'single_channel': ['arrays'], 'xarray_dict_area_def_to_numpy': ['xarray_dict', 'area_def'], 'xarray_dict_dict_to_xarray': ['xarray_dict_dict'], 'xarray_dict_to_xarray': ['xarray_dict'], 'xarray_dict_to_xarray_dict': ['xarray_dict'], 'xarray_to_numpy': ['xobj'], 'xarray_to_xarray': ['xobj', 'variables']}#
required_kwargs = {'channel_combination': [], 'list_numpy_to_numpy': [], 'rgb': [], 'scalar_to_scalar': ['value'], 'single_channel': [], 'xarray_dict_area_def_to_numpy': [], 'xarray_dict_dict_to_xarray': [], 'xarray_dict_to_xarray': [], 'xarray_dict_to_xarray_dict': [], 'xarray_to_numpy': [], 'xarray_to_xarray': []}#
class geoips.interfaces.class_based.algorithms.BaseAlgorithmPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS algorithm plugins.

Plugins with data_tree=False have their inputs / outputs automatically converted according to the family-specific rules defined in ALGORITHM_FAMILY_CONVERSIONS.

data_tree = False#

geoips.interfaces.class_based.colormappers module#

Colormappers interface class.

class geoips.interfaces.class_based.colormappers.BaseColormapperPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS colormapper plugins.

data_tree = False#
class geoips.interfaces.class_based.colormappers.ColormappersInterface[source]#

Bases: BaseClassInterface

Base interface class for colormappers plugins.

allowable_kwargs = {'matplotlib': ['data_range', 'cmap_name', 'ascii_path', 'create_colorbar', 'cbar_label', 'cbar_ticks', 'cbar_tick_labels', 'cbar_spacing', 'cbar_full_width', 'colorbar_kwargs', 'set_ticks_kwargs', 'set_label_kwargs']}#
name = 'colormappers'#
plugin_class#

alias of BaseColormapperPlugin

required_args: dict[str, list[str]] = {'matplotlib': []}#
required_kwargs = {'matplotlib': []}#

geoips.interfaces.class_based.coverage_checkers module#

Coverage checkers interface class.

class geoips.interfaces.class_based.coverage_checkers.BaseCoverageCheckerPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS coverage_checker plugins.

data_tree = False#
class geoips.interfaces.class_based.coverage_checkers.CoverageCheckersInterface[source]#

Bases: BaseClassInterface

Base interface class for coverage_checkers plugins.

allowable_kwargs = {'standard': {'area_def', 'radius_km'}}#
get_plugin_for_product(product, checker_field='coverage_checker')[source]#

Get plugin for product.

name = 'coverage_checkers'#
plugin_class#

alias of BaseCoverageCheckerPlugin

required_args: dict[str, list[str]] = {'standard': ['xarray_obj', 'variable_name']}#
required_kwargs = {'standard': {}}#

geoips.interfaces.class_based.databases module#

Databases interface class.

class geoips.interfaces.class_based.databases.BaseDatabasePlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS database plugins.

data_tree = False#
class geoips.interfaces.class_based.databases.DatabasesInterface[source]#

Bases: BaseClassInterface

Base interface class for databases plugins.

name = 'databases'#
plugin_class#

alias of BaseDatabasePlugin

required_args: dict[str, list[str]] = {'memusg_stats_to_table': [], 'overpass_dict_to_postgres_table': ['schema_name', 'table_name', 'overpass_dict'], 'overpass_dict_to_sqlite_table': ['overpass_dict', 'dbname'], 'query_overpass_table': ['window_start', 'window_end'], 'query_product_table': ['window_start', 'window_end'], 'schema_table_sector_opasses_to_postgres_table': ['schema_name', 'table_name', 'sector_name', 'sat_overpasses'], 'schema_table_storm_opasses_to_postgres_table': ['schema_name', 'table_name', 'storm_id', 'storm_time', 'satellite_overpasses'], 'storm_opasses_db_to_postgres_table': ['storm_id', 'storm_time', 'satellite_overpasses', 'dbname'], 'xarray_area_def_to_table': ['product_filename', 'xarray_obj', 'area_def'], 'xarray_dict_to_table': ['product_filename', 'xarray_dict']}#
required_kwargs = {'memusg_stats_to_table': [], 'overpass_dict_to_postgres_table': [], 'overpass_dict_to_sqlite_table': [], 'query_overpass_table': [], 'query_product_table': [], 'schema_table_sector_opasses_to_postgres_table': [], 'schema_table_storm_opasses_to_postgres_table': [], 'storm_opasses_db_to_postgres_table': [], 'xarray_area_def_to_table': [], 'xarray_dict_to_table': []}#

geoips.interfaces.class_based.filename_formatters module#

Filename formatters interface class.

class geoips.interfaces.class_based.filename_formatters.BaseFilenameFormatterPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS filename_formatter plugins.

data_tree = False#
class geoips.interfaces.class_based.filename_formatters.FilenameFormattersInterface[source]#

Bases: BaseClassInterface

Base interface class for filename_formatters plugins.

name = 'filename_formatters'#
plugin_class#

alias of BaseFilenameFormatterPlugin

required_args: dict[str, list[str]] = {'data': ['area_def', 'xarray_obj', 'product_names'], 'standard': ['area_def', 'xarray_obj', 'product_name'], 'standard_metadata': ['area_def', 'xarray_obj', 'product_filename'], 'xarray_area_product_to_filename': ['xarray_obj', 'area_def', 'product_name'], 'xarray_metadata_to_filename': ['xarray_obj']}#
required_kwargs = {'data': ['coverage', 'output_type', 'basedir'], 'standard': ['coverage', 'output_type', 'basedir'], 'standard_metadata': ['metadata_dir', 'metadata_type', 'basedir'], 'xarray_area_product_to_filename': ['output_type', 'basedir', 'extra_field'], 'xarray_metadata_to_filename': ['extension', 'basedir']}#
class geoips.interfaces.class_based.filename_formatters.WindsFilenameFormatterPlugin(module=None)[source]#

Bases: BaseFilenameFormatterPlugin

Base class for wind-based filename_formatter plugins.

assemble_windspeeds_text_full_fname(basedir, source_name, platform_name, data_provider, product_datetime, dt_format='%Y%m%d.%H%M', extension='.txt', creation_time=None)[source]#

Produce full output product path using product / sensor specifications.

Parameters:

  • basedir (str) – base directory

  • source_name (str) – Name of source (sensor)

  • platform_name (str) – Name of platform (satellite)

  • data_provider (str) – Name of data provider

  • product_datetime (datetime.datetime) – Start time of data used to generate product

  • dt_format (str, default="%Y%m%d.%H%M") – Format used to display product_datetime within filename

  • extension (str, default=".txt") – File extension, specifying type.

  • creation_time (datetime.datetime, default=None) – Include given creation_time of file in filename If None, do not include creation time.

Returns:

full path of output filename of the format:

<basedir>/<source_name>_<data_provider>_<platform_name>_ surface_winds_<YYYYMMDDHHMN>

Return type:

str

Examples

>>> startdt = datetime.strptime('20200216T001412', '%Y%m%dT%H%M%S')
>>> assemble_windspeeds_text_full_fname(
...     '/outdir',
...     'smap-spd',
...     'smap',
...     'remss',
...     startdt,
...     '%Y%m%d'
...     )
'/outdir/smap-spd_remss_smap_surface_winds_20200216'

geoips.interfaces.class_based.interpolators module#

Interpolators interface class.

class geoips.interfaces.class_based.interpolators.BaseInterpolatorPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS interpolator plugins.

data_tree = False#
class geoips.interfaces.class_based.interpolators.InterpolatorsInterface[source]#

Bases: BaseClassInterface

Base interface class for interpolators plugins.

name = 'interpolators'#
plugin_class#

alias of BaseInterpolatorPlugin

required_args: dict[str, list[str]] = {'2d': ['area_def', 'input_xarray', 'output_xarray', 'varlist'], 'grid': ['area_def', 'input_xarray', 'output_xarray', 'varlist']}#
required_kwargs = {'2d': ['array_num'], 'grid': ['array_num']}#

geoips.interfaces.class_based.output_checkers module#

Output Checkers interface class.

class geoips.interfaces.class_based.output_checkers.BaseOutputCheckerPlugin(module=None)[source]#

Bases: BaseClassPlugin

Output Checkers Base Plugin for comparing data outputs.

compare_outputs(compare_path, output_products, **kwargs)[source]#

Compare the “correct” imagery found the list of current output_products.

Compares files produced in the current processing run with the list of “correct” files contained in “compare_path”.

Parameters:

  • compare_path (str) – Path to directory of “correct” products - filenames must match output_products

  • output_products (list of str) – List of strings of current output products, to compare with products in compare_path

  • kwargs (dict) – Dictionary containing kwargs for comparing products. This gets passed through to the “test_products” method.

Returns:

Binary code: 0 if all comparisons were completed successfully.

Return type:

int

data_tree = False#
get_compare_products(compare_path)[source]#

Get dictionary of compare products from the compare_path.

Parameters:

compare_path (str) – Full path to the comparison directory to populate compare_products dictionary.

Returns:

  • compare_products (dict) – Dictionary of products found in the test comparison directory. The key of the compare_products dictionary is the filename that is stored in the comparison directory (ie, this may be a gzipped file).

    • stored_comparison is the actual name stored in the comparison test dir

    • gunzipped comparison is the gunzipped version of the file, or None if file was not gzipped

    • file_for_comparison is “compare_product” if the file is not gzipped, and “gunzipped_comparison” if file is gzipped.

  • remove_temp_files (list) – List of full paths to files that must be removed after processing (gunzipped files in GEOIPS_OUTDIRS scratch directory)

get_out_diff_fname(compare_product, output_product, ext=None, flag=None)[source]#

Obtain the filename for output and comparison product diff.

Parameters:

  • compare_product (str) – Full path to product filename in the comparison directory

  • output_product (str) – Full path to product filename in the current output directory

  • ext (str, default=None) – Extension to use as an alternative to the original file extension

  • flag (str, default=None) – Additional identifying string to include in output diff filename

Returns:

out_diff_fname – Full path to output diff file.

Return type:

str

gunzip_product(fname, is_comparison_product=False, clobber=False)[source]#

Gunzip file fname.

Parameters:

  • fname (str) – File to gunzip.

  • is_comparison_product (bool, default False) –

    • If True, set to $GEOIPS_OUTDIRS/gunzipped_products/comparison_products

    • If False, set to $GEOIPS_OUTDIRS/gunzipped_products/output_products

  • clobber (bool, default False) – If True, overwrite existing file, else do nothing.

Returns:

Filename after gunzipping

Return type:

str

is_gz(fname)[source]#

Check if fname is a gzip file.

Parameters:

fname (str) – Name of file to check.

Returns:

True if it is a gz file, False otherwise.

Return type:

bool

perform_comparisons(output_products, compare_products, **kwargs)[source]#

Get the final output products list from the original list of outputs.

Parameters:

  • output_products (list) – This is the original list of output products, exactly as they are produced during processing. If a product is gzipped during processing, the output_products list will include the gzipped file.

  • compare_products (dict) –

    This is a dictionary containing all of the comparison products available in the test outputs directory.

    • Keys: basename of actual output product as created during run (ie, gzipped if applicable)

      • key: “file_for_comparison”, value: full path to file that is used when comparing against the current output (ie, could be temporary gunzip location).

      • key: “gunzipped_comparison”, value: None if original output file was not gzipped, full path to gunzipped file if original output file was gzipped.

      • key: “stored_comparison”, value: full path to file that is stored in the tests outputs location (ie, could be gzipped)

  • kwargs (dict) – keyword arguments to pass directly through to test products function.

Returns:

  • output_products_for_comparison (list) – List of full paths to files that will be used for comparisons. Ie, if the output product was gzipped, this would be the full path to the gunzipped file in the temporary location.

  • missingcomps (list) – List of full paths to files that were found in the current run, and NOT found in the comparison directory.

  • goodcomps (list) – List of full paths to files that had good comparisons.

  • badcomps (list) – List of full paths to files that had bad comparisons between the current output products and the test comparison outputs.

  • compare_strings (list) – List of strings included in log output for each type of comparison.

  • remove_temp_files (list) – List of full paths to files that should be removed from temporary scratch directory.

test_products(output_product, compare_product, goodcomps, badcomps, compare_strings, **kwargs)[source]#

Test output_product against “good” product stored in “compare_path”.

Parameters:

  • output_product (str) –

    • Full path to current output product

  • compare_product (str) –

    • Full path to “good” comparison product

  • goodcomps (list of str) –

    • List of full paths to all “good” successful comparisons (output and compare images match)

    • Each str is prepended with a “compare_string” tag to identify which comparison type was performed.

  • badcomps (list of str) –

    • List of full paths to all “bad” unsuccessful comparisons (output and compare images differ)

    • Each str is prepended with a “compare_string” tag to identify which comparison type was performed.

  • compare_strings (list of str) –

    • List of all comparison “tags” included in goodcomps and badcomps lists.

    • This list is used to remove the comparison tags from goodcomps and badcomps to retrieve only the file path.

  • kwargs (dict) – Additional arguments to pass through to “outputs_match” method.

Returns:

  • goodcomps (list of str) – All current good comparisons appended to the list passed in.

  • badcomps (list of str) – All current bad comparisons appended to the list passed in.

  • compare_strings (list of str) – All current comparison “tags” added to the list passed in.

Raises:

TypeError – Raised when current output product does not have an associated comparison test defined.

class geoips.interfaces.class_based.output_checkers.OutputCheckersInterface[source]#

Bases: BaseClassInterface

Base interface class for output_checkers plugins.

get_plugin(name, rebuild_registries=None)[source]#

Return the output checker plugin corresponding to checker_name.

Parameters:

  • name (str) –

    • The name the desired plugin.

  • rebuild_registries (bool (default=None)) –

    • Whether or not to rebuild the registries if get_plugin fails. If set to None, default to what we have set in geoips.filenames.base_paths, which defaults to True. If specified, use the input value of rebuild_registries, which should be a boolean value. If rebuild registries is true and get_plugin fails, rebuild the plugin registry, call then call get_plugin once more with rebuild_registries toggled off, so it only gets rebuilt once.

identify_checker(filename, checker_override_name=None)[source]#

Identify the correct output checker plugin and return its name.

Parameters:

  • filename (str) –

    • The name of the file to be checked against a comparison output.

  • checker_override_name (str (default=None)) –

    • If set, specifies an output checker that should be used for the input, instead of trying to discover it from the file name. If the requested output checker is not found, fall back on trying to discover it from the file name.

Returns:

  • The name of the discovered output checker.

Return type:

string

name = 'output_checkers'#
plugin_class#

alias of BaseOutputCheckerPlugin

required_args: dict[str, list[str]] = {'standard': {}}#
required_kwargs = {'standard': {}}#
valid_plugin(plugin)[source]#

Check the validity of the supplied output_checker plugin.

geoips.interfaces.class_based.output_checkers.get_missing_products(output_products_for_comparison, compare_products)[source]#

Get list of missing products from compare_products_dictionary.

Parameters:

  • output_products_for_comparison (list) – List of output products from the current run.

  • compare_products (dict) – Dictionary of products found in the test comparison directory. Note the key of the compare_products dictionary is the filename that is stored in the comparison directory (ie, this may be a gzipped file). We must use the “gunzipped_comparison”, “stored_comparison”, and “file_for_comparison” keys within that dictionary to ensure we are accessing the correct versions of the files.

Returns:

missingproducts – List of products that were missing from the current run.

Return type:

list

geoips.interfaces.class_based.output_checkers.gunzip_product(fname, is_comparison_product=False, clobber=False)[source]#

Gunzip file fname.

Parameters:

  • fname (str) – File to gunzip.

  • is_comparison_product (bool, default False) –

    • If True, set to $GEOIPS_OUTDIRS/gunzipped_products/comparison_products

    • If False, set to $GEOIPS_OUTDIRS/gunzipped_products/output_products

  • clobber (bool, default False) – If True, overwrite existing file, else do nothing.

Returns:

Filename after gunzipping

Return type:

str

geoips.interfaces.class_based.output_checkers.is_gz(fname)[source]#

Check if fname is a gzip file.

Parameters:

fname (str) – Name of file to check.

Returns:

True if it is a gz file, False otherwise.

Return type:

bool

geoips.interfaces.class_based.output_checkers.write_bad_comparisons_to_file(badcomps, compare_products, compare_strings, diffdir)[source]#

Write text file with cp commands to update products with bad comparisons.

Write a text file that can be sourced to replace test output comparison files with the most recent outputs when the output checker returns a bad comparisons.

When GeoIPS is updated in a way that changes the imagery and those changes are expected, the integration test image comparisons will fail. This function writes a file named cp_BADCOMPARES.txt which can be sourced to copy the new comparison files into place, overwriting the previous comparison files. This allows easily updating the test comparison outputs when code updates make expected changes to the output imagery.

We must ensure we also gzip the comparison product in the test outputs directory after copying if the original test comparison output file was gzipped. If applicable, a gzip command will also be added to cp_BADCOMPARES.txt.

Parameters:

  • badcomps (list) – List of files that returned a “BAD COMPARISON” from the output checker. These are the full paths to the files produced during the most recent run.

  • compare_products (dict) – dictionary of comparison products - including the original comparison product file name, the gunzipped product file path if applicable, and the path to the file that should be used for comparison against “badcomps” files.

  • compare_strings (list of str) –

    List of all comparison “tags” included in goodcomps and badcomps lists.

    • This list is used to remove the comparison tags from goodcomps and badcomps to retrieve only the file path.

  • diffdir (str) – Full path to write “cp_BADCOMPARES.txt” file. NOTE: diffdir currently also is used to specify the path to the output files. Eventually we should pass in 2 separate paths, one to specify where to write the diff files, and a separate path indicating where to write the updated test output comparison files.

Returns:

Binary code: 0 if all comparisons were completed successfully, non-zero if there were bad comparisons.

Return type:

int

geoips.interfaces.class_based.output_checkers.write_good_comparisons_to_file(goodcomps, compare_strings, diffdir)[source]#

Write all good comparisons to a file for easy collection of outputs.

During integration tests, if the current product output matches the test output stored in the test comparison directories, the output checkers will return 0 as a GOOD COMPARE. These outputs do not need to be updated in the test comparison directories, since the current products and test comparison outputs match within tolerance.

The developer may desire copying these files to a single location convenience, however.

This function writes a file named cptest_GOODCOMPARE.txt which can be sourced to copy all products that were identical to the current output products to a common location, for convenience. This does not impact any files stored in the test comparison directories.

Parameters:

  • goodcomps (list) – List of files that returned a “GOOD COMPARE” from the output checker. These are the full paths to the files produced during the most recent run.

  • compare_strings (list of str) –

    List of all comparison “tags” included in goodcomps and badcomps lists.

    • This list is used to remove the comparison tags from goodcomps and badcomps to retrieve only the file path when writing the cp commands.

  • diffdir (str) – Full path to write “cptest_GOODCOMPARE.txt” file NOTE: diffdir currently also is used to specify the path to the output files. Eventually we should pass in 2 separate paths, one to specify where to write the diff files, and a separate path indicating where to write the updated test output comparison files.

Returns:

Binary code: 0 for good comparisons

Return type:

int

geoips.interfaces.class_based.output_checkers.write_missing_comparisons_to_file(missingcomps, diffdir)[source]#

Write text file with cp commands to add products to comparison directory.

Write a text file containing cp commands that can be sourced to add new test output comparison files to the test comparison directories that were found in the most recent output and were not previously in the test comparison directory.

When GeoIPS is updated in a way that intentionally causes a new output product, it will cause the integration test comparison to fail with a MISSING COMPARE error (ie, a file exists in the current output product path, but does not exist in the test comparison directory).

When this is intended behaviour due to recent code updates, we must add that new product to the test output comparison directories from the current output path.

This function writes a file named cp_MISSINGCOMPARE.txt which can be sourced to copy the appropriate files from the current output product path into the test comparison directory. This allows easily updating the test comparison outputs when code updates make expected changes to the output products.

Parameters:

  • missingcomps (list) – List of files that returned a “MISSING COMPARE” from the output checker. These are the full paths to the files produced during the most recent run.

  • diffdir (str) – Full path to write “rm_MISSINGCOMPARE.txt” file NOTE: diffdir currently also is used to specify the path to the output files. Eventually we should pass in 2 separate paths, one to specify where to write the diff files, and a separate path indicating where to write the updated test output comparison files.

Returns:

Binary code: 0 if all comparisons were completed successfully, non-zero if there were missing comparisons.

Return type:

int

geoips.interfaces.class_based.output_checkers.write_missing_products_to_file(missingproducts, compare_products, diffdir)[source]#

Write text file with rm commands to remove products from comparison directory.

Write a text file containing rm commands that can be sourced to remove existing test output comparison files from the test comparison directories that were NOT found in the most recent output.

When GeoIPS is updated in a way that intentionally causes a previously expected output to no longer be produced, it will cause the integration test comparison to fail with a MISSING PRODUCT error (ie, a file exists in the test comparison directory, but is not a current output product).

When this is intended behaviour due to recent code updates, we must remove that product from the test output comparison directories.

This function writes a file named rm_MISSINGPRODUCTS.txt which can be sourced to remove the files from the test comparison directory that were NOT found in the list of current output products. This allows easily updating the test comparison outputs when code updates make expected changes to the output products.

Parameters:

  • missingproducts (list) – List of files that returned a “MISSING PRODUCT” from the output checker. These are the full paths to the files produced during the most recent run.

  • compare_products (dict) – dictionary of comparison products - including the original comparison product file name, the gunzipped product file path if applicable, and the path to the file that should be used for comparison against “missingproducts” files.

  • diffdir (str) – Full path to write “rm_MISSINGPRODUCTS.txt” file NOTE: diffdir currently also is used to specify the path to the output files. Eventually we should pass in 2 separate paths, one to specify where to write the diff files, and a separate path indicating where to write the updated test output comparison files.

Returns:

Binary code: 0 if all comparisons were completed successfully, non-zero if there were missing products.

Return type:

int

geoips.interfaces.class_based.output_checkers.write_remove_temp_files_to_file(remove_temp_files, diffdir)[source]#

Write text file with rm commands to remove temporary files.

Write a text file rm_TEMPFILES.txt that can be sourced to remove temporary files that were created in the $GEOIPS_OUTDIRS scratch directory during the output comparison checks during integration tests. This currently includes temporary gunzipped versions of products (ie, if a product was gzipped during processing, then gunzipped to a temporary location during the comparison tests, it will get added to the remove_temp_files list), and may include additional temporary files in the future.

The gunzipped files are not auto-deleted during the comparison process, because they must be manually copied to the tets outputs directory after processing is complete, so we do not remove the files at runtime.

Parameters:

  • remove_temp_files (list) – List of files that should be written to rm_TEMPFILES.txt for deletion. This currently includes auto-gunzipped products, and potentially additional temporary files in the future.

  • diffdir (str) – Full path to write rm_TEMPFILES.txt file NOTE: diffdir currently also is used to specify the path to the output files. Eventually we should pass in 2 separate paths, one to specify where to write the diff files, and a separate path indicating where to write the updated test output comparison files.

Returns:

Binary code: 0 for successfully written rm_TEMPFILES.txt file.

Return type:

int

geoips.interfaces.class_based.output_formatters module#

Output formatters interface class.

class geoips.interfaces.class_based.output_formatters.BaseOutputFormatterPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS output_formatter plugins.

Plugins with data_tree=False have their inputs / outputs automatically converted according to the family-specific rules defined in OUTPUT_FORMATTER_FAMILY_CONVERSIONS.

data_tree = False#
update_sector_info_with_default_metadata(area_def, xarray_obj, product_filename=None, metadata_filename=None)[source]#

Update sector info found in “area_def” with standard metadata output.

This function is used by metadata_tc output formatter as well for updating the sector_info with these additional default metadata fields. We should not filter out non-default metadata here, since metadata_tc uses this function.

Parameters:

  • area_def (AreaDefinition) – Pyresample AreaDefinition of sector information

  • xarray_obj (xarray.Dataset) – xarray Dataset object that was used to produce product

  • product_filename (str) – Full path to full product filename that this YAML file refers to

Returns:

sector_info dict with standard metadata added

  • bounding box

  • product filename with wildcards

  • basename of original source filenames

Return type:

dict

class geoips.interfaces.class_based.output_formatters.NetcdfOutputFormatterPlugin(module=None)[source]#

Bases: BaseOutputFormatterPlugin

Base class for netcdf-based output formatter plugins.

clean_attr_for_netcdf(xobj, attr)[source]#

Check xarray attributes.

write_xarray_netcdf(xarray_obj, ncdf_fname, clobber=False, use_compression=True, compression_kwargs=None)[source]#

Write out xarray_obj to netcdf file named ncdf_fname.

class geoips.interfaces.class_based.output_formatters.OutputFormattersInterface[source]#

Bases: BaseClassInterface

Base interface class for output_formatters plugins.

name = 'output_formatters'#
plugin_class#

alias of BaseOutputFormatterPlugin

required_args: dict[str, list[str]] = {'image': ['xarray_obj', 'area_def', 'product_name', 'output_fnames'], 'image_multi': ['xarray_obj', 'area_def', 'product_names', 'output_fnames', 'mpl_colors_info'], 'image_overlay': ['xarray_obj', 'area_def', 'product_name', 'output_fnames'], 'standard_metadata': ['xarray_obj', 'area_def', 'metadata_yaml_filename', 'product_filename'], 'unprojected': ['xarray_obj', 'product_name', 'output_fnames'], 'xarray_data': ['xarray_obj', 'product_names', 'output_fnames'], 'xrdict_area_product_outfnames_to_outlist': ['xarray_dict', 'area_def', 'product_name', 'output_fnames'], 'xrdict_area_product_to_outlist': ['xarray_dict', 'area_def', 'product_name'], 'xrdict_area_varlist_to_outlist': ['xarray_dict', 'area_def', 'varlist'], 'xrdict_to_outlist': ['xarray_dict'], 'xrdict_varlist_outfnames_to_outlist': ['xarray_dict', 'varlist', 'output_fnames']}#
required_kwargs = {'image': ['product_name_title', 'mpl_colors_info', 'existing_image'], 'image_multi': ['product_name_titles'], 'image_overlay': ['product_name_title', 'clean_fname', 'mpl_colors_info', 'clean_fname', 'feature_annotator', 'gridline_annotator', 'clean_fname', 'product_datatype_title', 'clean_fname', 'bg_data', 'bg_mpl_colors_info', 'clean_fname', 'bg_xarray', 'bg_product_name_title', 'bg_datatype_title', 'clean_fname', 'remove_duplicate_minrange'], 'standard_metadata': ['metadata_dir', 'basedir', 'output_dict'], 'unprojected': ['product_name_title', 'mpl_colors_info'], 'xarray_data': [], 'xarray_dict_data': ['append', 'overwrite'], 'xarray_dict_to_image': [], 'xrdict_area_product_outfnames_to_outlist': [], 'xrdict_area_product_to_outlist': [], 'xrdict_area_varlist_to_outlist': [], 'xrdict_to_outlist': [], 'xrdict_varlist_outfnames_to_outlist': []}#
class geoips.interfaces.class_based.output_formatters.WindbarbOutputFormatterPlugin(module=None)[source]#

Bases: BaseOutputFormatterPlugin

Base class for windbarb-based output formatter plugins.

assign_height_levels(windbarb_data_dict, pressure_range_dict)[source]#

Assing derived motion winds to specified height levels.

Using the pressure associated with each retrieved wind observation, assign to a specified level (e.g. Low/Mid/High) based on predefined pressure ranges. Each pressure level is assigned an integer, and any unassigned are set to 0

Parameters:

  • formatted_data_dict (dict) – Dictionary holding DMW data - must include a pressure key

  • pressure_range_dict (dict) – Dictionary specifying pressure range for each defined level. e.g. {“Low”: [701, 1013.25], “Mid”: [401, 700], “High”: [0, 400]}

Returns:

Array of assigned level numbers, and list of associated labels that can be used to set the ticks on a colorbar

Return type:

tuple

format_windbarb_data(xarray_obj, product_name)[source]#

Format windbarb data before plotting.

output_clean_windbarbs(area_def, clean_fnames, mpl_colors_info, image_datetime, formatted_data_dict, fig=None, main_ax=None, mapobj=None, barb_color_variable='speed')[source]#

Plot and save “clean” windbarb imagery.

No background imagery, coastlines, gridlines, titles, etc.

Returns:

Full paths to all resulting output files.

Return type:

list of str

plot_barbs(main_ax, mapobj, mpl_colors_info, formatted_data_dict, barb_color_variable='speed')[source]#

Plot windbarbs on matplotlib figure.

geoips.interfaces.class_based.procflows module#

Procflows interface class.

class geoips.interfaces.class_based.procflows.BaseProcflowPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS procflow plugins.

data_tree = True#
class geoips.interfaces.class_based.procflows.ProcflowsInterface[source]#

Bases: BaseClassInterface

Base interface class for procflows plugins.

name = 'procflows'#
plugin_class#

alias of BaseProcflowPlugin

required_args: dict[str, list[str]] = {'order_based': ['workflow_spec', 'filenames'], 'standard': ['fnames']}#
required_kwargs = {'order_based': [], 'standard': ['command_line_args']}#

geoips.interfaces.class_based.readers module#

Readers interface class.

class geoips.interfaces.class_based.readers.BaseAbiReaderPlugin(module=None)[source]#

Bases: BaseReaderPlugin

Base class for ABI-based readers.

BADVALS = {'Conditional': -999.8, 'No_Value': -999.6, 'Off_Of_Disk': -999.9, 'Out_Of_Valid_Range': -999.7, 'Uninitialized': -9999.9}#
get_latitude_longitude(metadata, BADVALS, sect=None, geolocation_cache_backend='memmap', chunk_size=None, resource_tracker=None)[source]#

Get latitudes and longitudes.

This routine accepts a dictionary containing metadata as read from a NCDF4 format file, and returns latitudes and longitudes for a full disk.

class geoips.interfaces.class_based.readers.BaseReaderPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS reader plugins.

data_tree = False#
class geoips.interfaces.class_based.readers.ReadersInterface[source]#

Bases: BaseClassInterface

Base interface class for readers plugins.

call_files_and_get_top_level_metadata(fnames, all_metadata, call_single_file_func, metadata_only=False, chans=None, area_def=None, self_register=False, **kwargs)[source]#

Read in data from a list of filenames.

Parameters:

  • fnames (list) –

    • List of strings, full paths to files

  • all_metadata (dict) –

    • Dictionary of metadata from all files in ‘fnames’

  • read_single_time_func (python function) –

    • Function which can be used to read a single scan time of files from a reader plugin.

    • Most likely named ‘call_single_time’.

  • metadata_only (bool, default=False) –

    • Return before actually reading data if True

  • chans (list of str, default=None) –

    • List of desired channels (skip unneeded variables as needed).

    • Include all channels if None.

  • area_def (pyresample.AreaDefinition, default=None) –

    • Specify region to read

    • Read all data if None.

  • self_register (str or bool, default=False) –

    • register all data to the specified dataset id (as specified in the return dictionary keys).

    • Read multiple resolutions of data if False.

Returns:

  • dictionary of xarray.Dataset objects with required Variables and Attributes.

  • Dictionary keys can be any descriptive dataset ids.

Return type:

dict of xarray.Datasets

See also

Xarray Standards

Additional information regarding required attributes and variables for GeoIPS-formatted xarray Datasets.

concatenate_metadata(all_metadata)[source]#

Merge together metadata sourced from a list of files into one dictionary.

Where the structure of the merged metadata is a nested dictionary of metadata.

Ie. (xarray_obj has no data and is merely just a container for metadata): {“METADATA”: xobj.source_file_attributes: {fname: xobj, …, “fnamex”: xobj}}

Parameters:

all_metadata (list of xarray.Datasets) –

  • The incoming metadata from X number of files

Returns:

md –

  • All metadata merged into a dictionary of xarray Datasets

Return type:

dict of xarray Datasets

name = 'readers'#
plugin_class#

alias of BaseReaderPlugin

read_data_to_xarray_dict(fnames, read_single_time_func, metadata_only=False, chans=None, area_def=None, self_register=False, **kwargs)[source]#

Read in data potentially from multiple scan times into an xarray dict.

This function does not require that you provide multiple scan times, but allows for that in the case those are provided.

Call this with information specific to your reader to generate a dictionary of xarray datasets created from the data provided in ‘fnames’.

Parameters:

  • fnames (list) –

    • List of strings, full paths to files

  • all_metadata (dict) –

    • Dictionary of metadata from all files in ‘fnames’

  • read_single_time_func (python function) –

    • Function which can be used to read a single scan time of files from a reader plugin.

    • Most likely named ‘call_single_time’.

  • metadata_only (bool, default=False) –

    • Return before actually reading data if True

  • chans (list of str, default=None) –

    • List of desired channels (skip unneeded variables as needed).

    • Include all channels if None.

  • area_def (pyresample.AreaDefinition, default=None) –

    • Specify region to read

    • Read all data if None.

  • self_register (str or bool, default=False) –

    • register all data to the specified dataset id (as specified in the return dictionary keys).

    • Read multiple resolutions of data if False.

Returns:

  • dictionary of xarray.Dataset objects with required Variables and Attributes.

  • Dictionary keys can be any descriptive dataset ids.

Return type:

dict of xarray.Datasets

See also

Xarray Standards

Additional information regarding required attributes and variables for GeoIPS-formatted xarray Datasets.

required_args: dict[str, list[str]] = {'standard': ['fnames']}#
required_kwargs = {'standard': ['metadata_only', 'chans', 'area_def', 'self_register']}#

geoips.interfaces.class_based.sector_adjusters module#

Sector adjusters interface class.

class geoips.interfaces.class_based.sector_adjusters.BaseSectorAdjusterPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS sector_adjuster plugins.

data_tree = False#
class geoips.interfaces.class_based.sector_adjusters.SectorAdjustersInterface[source]#

Bases: BaseClassInterface

Base interface class for sector_adjusters plugins.

name = 'sector_adjusters'#
plugin_class#

alias of BaseSectorAdjusterPlugin

required_args: dict[str, list[str]] = {'list_xarray_list_variables_to_area_def_out_fnames': ['xobjs', 'area_def', 'variables']}#
required_kwargs = {'list_xarray_list_variables_to_area_def_out_fnames': ['recenter_variables']}#

geoips.interfaces.class_based.sector_metadata_generators module#

Sector metadata generators interface class.

class geoips.interfaces.class_based.sector_metadata_generators.BaseSectorMetadataGeneratorPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS sector_metadata_generator plugins.

data_tree = False#
class geoips.interfaces.class_based.sector_metadata_generators.DeckSectorMetaGeneratorPlugin(module=None)[source]#

Bases: BaseSectorMetadataGeneratorPlugin

Base class for GeoIPS deck-based sector_metadata_generator plugins.

assemble_invest_storm_id(storm_basin, invest_number, storm_year, storm_start_datetime)[source]#

Assemble invest storm ID from basin, invest number, year, and start datetime.

Of format bbNNyyyyYYYYMMDD, where

  • bb is the storm basin (lower case)

  • NN is the 2-digit invest number (9x)

  • yyyy is the storm year

  • YYYYMMDDHH is the storm start datetime.

Note Invest storm ids include the storm start datetime, but numbered storm ids do not.

  • numbered storm bbNNyyyy

  • invest storm bbNNyyyyYYYYMMDDHH (ugly, but consistent with no delimiters)

assemble_numbered_storm_id(storm_basin, storm_number, storm_year)[source]#

Assemble numbered storm ID from storm basin, number, and year.

Of format bbNNyyyy, where

  • bb is the storm basin (lower case)

  • NN is the 2-digit invest number (9x)

  • yyyy is the storm year.

Note Invest storm ids include the storm start datetime, but numbered storm ids do not.

  • numbered storm bbNNyyyy

  • invest storm bbNNyyyyYYYYMMDDHH (ugly, but consistent with no delimiters)

class geoips.interfaces.class_based.sector_metadata_generators.SectorMetadataGeneratorsInterface[source]#

Bases: BaseClassInterface

Base interface class for sector_metadata_generators plugins.

name = 'sector_metadata_generators'#
plugin_class#

alias of BaseSectorMetadataGeneratorPlugin

required_args: dict[str, list[str]] = {'tc': ['trackfile_name'], 'volc': ['trackfile_name']}#
required_kwargs = {'tc': [], 'volc': []}#

geoips.interfaces.class_based.sector_spec_generators module#

Sector spec generators interface class.

class geoips.interfaces.class_based.sector_spec_generators.BaseSectorSpecGeneratorPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS sector_spec_generator plugins.

data_tree = False#
class geoips.interfaces.class_based.sector_spec_generators.SectorSpecGeneratorsInterface[source]#

Bases: BaseClassInterface

Base interface class for sector_spec_generators plugins.

name = 'sector_spec_generators'#
plugin_class#

alias of BaseSectorSpecGeneratorPlugin

required_args: dict[str, list[str]] = {'area_definition': []}#
required_kwargs = {'area_definition': []}#

geoips.interfaces.class_based.title_formatters module#

Title formatters interface class.

class geoips.interfaces.class_based.title_formatters.BaseTitleFormatterPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS title_formatter plugins.

data_tree = False#
class geoips.interfaces.class_based.title_formatters.TitleFormattersInterface[source]#

Bases: BaseClassInterface

Base interface class for title_formatters plugins.

name = 'title_formatters'#
plugin_class#

alias of BaseTitleFormatterPlugin

required_args: dict[str, list[str]] = {'standard': []}#
required_kwargs = {'standard': []}#

geoips.interfaces.class_based.validators module#

Validators interface module.

class geoips.interfaces.class_based.validators.BaseValidatorPlugin(module=None)[source]#

Bases: BaseClassPlugin

Base class for GeoIPS validator plugins.

class geoips.interfaces.class_based.validators.ValidatorsInterface[source]#

Bases: BaseClassInterface

Base interface class for validators plugins.

allowable_kwargs = {'standard': {'vars_to_validate'}}#
name = 'validators'#
plugin_class#

alias of BaseValidatorPlugin

required_args: dict[str, list[str]] = {'standard': ['xarray_obj', 'truth_xarray_obj']}#
required_kwargs = {'standard': {}}#

geoips.interfaces.class_based.workflow module#

DataTree-spec Workflow composite executor.

The Workflow class is a non-registered runtime executor that loads a validated WorkflowSpecModel resolves plugins, builds a topological step order, and walks through the steps collecting downstream provenance into an xr.DataTree. Retention is governed by the Strategy pattern so the execution loop never branches on the policy name.

split steps fan a workflow out into one branch per scope (e.g. one per static sector in globals.sector_list), running an inline body sub-workflow per branch; join steps re-collect those branches. Conditional execution (when) is accepted by the schema but not yet implemented and raises NotImplementedError at runtime.

class geoips.interfaces.class_based.workflow.KeepAllPolicy(spec: WorkflowSpecModel)[source]#

Bases: RetentionPolicy

Never GC any step data.

can_gc(step_id: str, *, executed: set[str]) → bool[source]#

Return True if the step’s data_vars can be dropped.

class geoips.interfaces.class_based.workflow.KeepReferencedPolicy(spec: WorkflowSpecModel)[source]#

Bases: RetentionPolicy

GC when no remaining downstream step references this step.

can_gc(step_id: str, *, executed: set[str]) → bool[source]#

Return True if the step’s data_vars can be dropped.

class geoips.interfaces.class_based.workflow.RetentionPolicy(spec: WorkflowSpecModel)[source]#

Bases: ABC

Decides whether a step’s data variables may be garbage-collected.

Parameters:

spec (WorkflowSpecModel) – The workflow specification.

abstract can_gc(step_id: str, *, executed: set[str]) → bool[source]#

Return True if the step’s data_vars can be dropped.

class geoips.interfaces.class_based.workflow.StepProvenance(plugin_name: str, plugin_kind: str, start_time: str, end_time: str, arguments_hash: str, output_token: str, gc_status: str = 'kept')[source]#

Bases: object

Per-step provenance bundle stamped into a step node’s attrs.

arguments_hash: str#
end_time: str#
classmethod from_attrs(attrs: dict) → StepProvenance[source]#

Reconstruct from a node’s ds.attrs dict.

gc_status: str#
output_token: str#
plugin_kind: str#
plugin_name: str#
start_time: str#
class geoips.interfaces.class_based.workflow.Workflow(spec: WorkflowSpecModel, workflow_name: str)[source]#

Bases: object

Non-registered runtime executor for a validated workflow spec.

This is an orchestrator that co-ordinates registered plugins — it does not inherit from BaseClassPlugin and is not itself registerable. Plugins are resolved via geoips.interfaces at runtime.

Parameters:

  • spec (WorkflowSpecModel) – A pydantic-validated workflow specification.

  • workflow_name (str) – The workflow name (used as the root DataTree node name).

call(workflow_tree=None, *, filenames=None, **kwargs)[source]#

Execute the workflow steps in topological order.

Parameters:

  • workflow_tree (xr.DataTree or None) – A DataTree to execute into. If None, a new root is created.

  • filenames (list[str] or None) – Input filenames for reader steps.

  • **kwargs – Additional keyword arguments forwarded to plugins.

Returns:

The fully-populated workflow DataTree.

Return type:

xr.DataTree

Module contents#

Class based interfaces init file.

On this page
Show Source