ARC to WARC Migration with CDX Index and wayback rendering screenshot comparison

None

class FileExtFilter implements FileFilter {

	public FileExtFilter(String ext) {
		this.ext = ext;
	}

	public boolean accept(File file) {
		return file.getName().endsWith(ext);
	}

	String ext = null;
}

if (extension == void || extension.equals("")) {
	throw new RuntimeException(
			"The 'extension' parameter cannot be null.  Please enter a valid file extension.");
}

if (directory == void || directory.equals("")) {
	throw new RuntimeException(
			"The 'directory' parameter cannot be null.  Please enter a valid file directory.");
}

File dirObj = new File(directory);

if (!dirObj.exists()) {
	throw new RuntimeException("The 'directory' parameter specified:" + directory
			+ "does not exist.  Please enter a valid file directory.");
}

File[] fileObjList = dirObj.listFiles(new FileExtFilter(extension));
List filelist = new ArrayList();
for (int i = 0; i < fileObjList.length; i++) {
	filelist.add(fileObjList[i].getAbsolutePath());
}

Name	Description
input_directory	Local input directory containing ARC files
output_directory	Local output directory where the migrated WARC files will be stored

Name	Type	Description
arc2warc_migration_cli	externaltool
arc2warc_jar	stringconstant	Value /path/to/hawarp/arc2warc-migration-cli/target/arc2warc-migration-cli-1.0-jar-with-dependencies.jar
cdx_creator_arc	externaltool
list_arc_files	localworker	Script class FileExtFilter implements FileFilter { public FileExtFilter(String ext) { this.ext = ext; } public boolean accept(File file) { return file.getName().endsWith(ext); } String ext = null; } if (extension == void \|\| extension.equals("")) { throw new RuntimeException( "The 'extension' parameter cannot be null. Please enter a valid file extension."); } if (directory == void \|\| directory.equals("")) { throw new RuntimeException( "The 'directory' parameter cannot be null. Please enter a valid file directory."); } File dirObj = new File(directory); if (!dirObj.exists()) { throw new RuntimeException("The 'directory' parameter specified:" + directory + "does not exist. Please enter a valid file directory."); } File[] fileObjList = dirObj.listFiles(new FileExtFilter(extension)); List filelist = new ArrayList(); for (int i = 0; i < fileObjList.length; i++) { filelist.add(fileObjList[i].getAbsolutePath()); }
arc.gz	stringconstant	Value arc.gz
cdx_creator_warc	externaltool
cdx_creator_jar	stringconstant	Value /path/to/hawarp/cdx-creator/target/cdx-creator-1.0-jar-with-dependencies.jar
diff_cdx	externaltool
cdx_creator_config	stringconstant	Value /path/to/config.properties
prep_arc_render	externaltool
cleanup	externaltool
arc_render	externaltool
warc_render	externaltool
compare_snapshots	externaltool
csvdiff_install_dir	stringconstant	Value /path/to/csvdiff
wayback_pathindex_file	stringconstant	Value /path/to/waybackbasedir/path-index.txt
wayback_cdx_index	stringconstant	Value /path/to/waybackbasedir/cdx-index/index.cdx
prep_warc_render	externaltool
wayback_url	stringconstant	Value http://localhost:8080/wayback/
phantomjs_snapshot_js	stringconstant	Value /path/to/storepngsnapshot.js
remove_wayback_index	externaltool

Name	Description
diff_out
compare_snapshots_out

Source	Sink
arc2warc_jar:value	arc2warc_migration_cli:arc2warc_jar_path
output_directory	arc2warc_migration_cli:output_directory
list_arc_files:filelist	arc2warc_migration_cli:arc_input_file
arc2warc_migration_cli:STDOUT	cdx_creator_arc:inputfile
cdx_creator_config:value	cdx_creator_arc:configfilepath
cdx_creator_jar:value	cdx_creator_arc:cdx_creator_jar
output_directory	cdx_creator_arc:output_directory
input_directory	list_arc_files:directory
arc.gz:value	list_arc_files:extension
cdx_creator_jar:value	cdx_creator_warc:cdx_creator_jar
cdx_creator_config:value	cdx_creator_warc:configfilepath
arc2warc_migration_cli:STDOUT	cdx_creator_warc:inputfile
output_directory	cdx_creator_warc:output_directory
cdx_creator_arc:STDOUT	diff_cdx:cdx_arc
cdx_creator_warc:STDOUT	diff_cdx:cdx_warc
csvdiff_install_dir:value	diff_cdx:csvdiff_install_dir
list_arc_files:filelist	prep_arc_render:arc_input_file
cdx_creator_arc:STDOUT	prep_arc_render:arc_cdx_file
wayback_pathindex_file:value	prep_arc_render:wayback_pathindex_file
wayback_cdx_index:value	prep_arc_render:wayback_cdx_index
output_directory	cleanup:output_directory
wayback_cdx_index:value	cleanup:wayback_cdx_index
wayback_pathindex_file:value	cleanup:wayback_path_index
output_directory	arc_render:output_directory
wayback_cdx_index:value	arc_render:wayback_cdx_index
wayback_url:value	arc_render:wayback_url
phantomjs_snapshot_js:value	arc_render:phantomjs_snapshot_js
output_directory	warc_render:output_directory
wayback_cdx_index:value	warc_render:wayback_cdx_index
wayback_url:value	warc_render:wayback_url
phantomjs_snapshot_js:value	warc_render:phantomjs_snapshot_js
output_directory	compare_snapshots:output_directory
wayback_cdx_index:value	compare_snapshots:wayback_cdx_index
wayback_url:value	compare_snapshots:wayback_url
wayback_pathindex_file:value	prep_warc_render:wayback_pathindex_file
output_directory	prep_warc_render:output_directory
list_arc_files:filelist	prep_warc_render:arc_input_file
cdx_creator_warc:STDOUT	prep_warc_render:warc_cdx_file
wayback_cdx_index:value	prep_warc_render:wayback_cdxindex_file
wayback_pathindex_file:value	remove_wayback_index:wayback_path_index
wayback_cdx_index:value	remove_wayback_index:wayback_cdx_index
diff_cdx:STDOUT	diff_out
compare_snapshots:STDOUT	compare_snapshots_out

Controller	Target
remove_wayback_index	prep_warc_render
cdx_creator_arc	diff_cdx
arc_render	prep_warc_render
diff_cdx	prep_arc_render
arc_render	remove_wayback_index
prep_warc_render	warc_render
warc_render	compare_snapshots
prep_arc_render	arc_render
cdx_creator_warc	diff_cdx
cleanup	arc2warc_migration_cli

ARC to WARC Migration with CDX Index and wayback rendering screenshot comparison

Preview

Run

Run this Workflow in the Taverna Workbench...

Workflow Components

Value

Script

Value

Value

Value

Value

Value

Value

Value

Value

Reviews (0)

Comments (0)

Other workflows that use similar services (0)