Data Refinement Workflow v17

Created: 2012-04-11 10:08:25 Last updated: 2014-12-17 12:51:15

Download Workflow

This workflow brings together various sub-workflows in one integrated workflow. This includes, - Synonym Expansion / Occurrence Retrieval - BioSTIFF Data Selection Tool - Google Refine Data Quality Installation instructions for this workflow can be found here https://wiki.biovel.eu/display/BioVeL/Taxonomic+Data+Refinement+Workflow+Installation.

This workflow depends on DCWorkflow.jar, which must be installed in Taverna's local user lib directory.

Preview

Download as scalable diagram (SVG)

Run

Run this Workflow in the Taverna Workbench...

Option 1:

Copy and paste this link into File > 'Open workflow location...'
http://myexperiment.org/workflows/2874/download?version=5
[ More Info Expand ]

Workflow Components

Authors (0)

Titles (0)

Descriptions (0)

Dependencies (2)

Inputs (0)

Processors (9)

Name	Type	Description
Select_File	localworker	Script import java.awt.CardLayout; import java.awt.Image; import java.awt.Toolkit; import java.io.File; import java.util.HashMap; import java.util.Map; import javax.swing.ImageIcon; import javax.swing.JEditorPane; import javax.swing.JFileChooser; import javax.swing.JLabel; import javax.swing.JPanel; import javax.swing.filechooser.FileFilter; class FileExtFilter extends FileFilter { public FileExtFilter(String ext, String label, boolean includeDir) { this.ext = ext; this.label = label; this.includeDir = includeDir; } public String getDescription() { return this.label; } public boolean accept(File file) { if (file.isDirectory() && includeDir) { return true; } else { return file.getName().endsWith(this.ext); } } String ext, label; boolean includeDir; } if (title == void) { title = null; } if ((fileExtensions == void) \|\| (fileExtensions == null)) { fileExtensions = ""; } if ((fileExtLabels == void) \|\| (fileExtLabels == null)) { fileExtLabels = ""; } JFileChooser chooser = new JFileChooser(); chooser.setDialogTitle(title); String[] fileTypeList = fileExtensions.split(","); String[] filterLabelList = fileExtLabels.split(","); if (fileTypeList != null && filterLabelList != null && fileTypeList.length != filterLabelList.length) { throw new RuntimeException("The list of extensions and file filter labels must be the same length"); } // create the file filters for (int i = 0; i < fileTypeList.length; i++) { FileExtFilter filter = new FileExtFilter(fileTypeList[i], filterLabelList[i], true); chooser.setFileFilter(filter); } chooser.showOpenDialog(null); File file = chooser.getSelectedFile(); selectedFile = file.getAbsolutePath();
title_value	stringconstant	Value Choose input file
Read_Text_File	localworker	Script BufferedReader getReader (String fileUrl, String encoding) throws IOException { InputStreamReader reader; try { if (encoding == null) { reader = new FileReader(fileUrl); } else { reader = new InputStreamReader(new FileInputStream(fileUrl),encoding); } } catch (FileNotFoundException e) { // try a real URL instead URL url = new URL(fileUrl); if (encoding == null) { reader = new InputStreamReader (url.openStream()); } else { reader = new InputStreamReader (url.openStream(), encoding); } } return new BufferedReader(reader); } StringBuffer sb = new StringBuffer(4000); if (encoding == void) { encoding = null; } BufferedReader in = getReader(fileurl, encoding); String str; String lineEnding = System.getProperty("line.separator"); while ((str = in.readLine()) != null) { sb.append(str); sb.append(lineEnding); } in.close(); filecontents = sb.toString();
Data_Cleaning_Worklow_Loop	workflow
Merge_String_List_to_a_String	localworker	Script String seperatorString = "\n"; if (seperator != void) { seperatorString = seperator; } StringBuffer sb = new StringBuffer(); for (Iterator i = stringlist.iterator(); i.hasNext();) { String item = (String) i.next(); sb.append(item); if (i.hasNext()) { sb.append(seperatorString); } } concatenated = sb.toString();
no_separator	stringconstant	Value
Write_Text_File	localworker	Script writeOK = "false"; BufferedWriter out; if (encoding == void) { out = new BufferedWriter(new FileWriter(outputFile)); } else { out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(outputFile), encoding)); } out.write(filecontents); out.flush(); out.close(); writeOK = "true";
Select_Output_CSV_File	localworker	Script import java.awt.CardLayout; import java.awt.Image; import java.awt.Toolkit; import java.io.File; import java.util.HashMap; import java.util.Map; import javax.swing.ImageIcon; import javax.swing.JEditorPane; import javax.swing.JFileChooser; import javax.swing.JLabel; import javax.swing.JPanel; import javax.swing.filechooser.FileFilter; class FileExtFilter extends FileFilter { public FileExtFilter(String ext, String label, boolean includeDir) { this.ext = ext; this.label = label; this.includeDir = includeDir; } public String getDescription() { return this.label; } public boolean accept(File file) { if (file.isDirectory() && includeDir) { return true; } else { return file.getName().endsWith(this.ext); } } String ext, label; boolean includeDir; } if (title == void) { title = null; } if ((fileExtensions == void) \|\| (fileExtensions == null)) { fileExtensions = ""; } if ((fileExtLabels == void) \|\| (fileExtLabels == null)) { fileExtLabels = ""; } JFileChooser chooser = new JFileChooser(); chooser.setDialogTitle(title); String[] fileTypeList = fileExtensions.split(","); String[] filterLabelList = fileExtLabels.split(","); if (fileTypeList != null && filterLabelList != null && fileTypeList.length != filterLabelList.length) { throw new RuntimeException("The list of extensions and file filter labels must be the same length"); } // create the file filters for (int i = 0; i < fileTypeList.length; i++) { FileExtFilter filter = new FileExtFilter(fileTypeList[i], filterLabelList[i], true); chooser.setFileFilter(filter); } chooser.showOpenDialog(null); File file = chooser.getSelectedFile(); selectedFile = file.getAbsolutePath();
output_filechooser_title	stringconstant	Value Choose Output CSV File

Beanshells (35)

Name	Inputs	Outputs
GBIF_Agreement_Conditional	gbif_agreement_answer	gbif_agreement_conditional
passthrough	csvin	csvout
Parse_Job	jsonStr	jsonErr jobID
Format_Options	jobID	options
Delete_Data_Options	projectID	options
Export_Data_Conditional	gref_answer	save_true cancel_true
End_Workflow	csvData	csv_output
OccTargetConditional	sciNameList	gbifList slwList gbifChosen
Occ_Credit_Checker	gbif_agreement_conditional gbif_names_list slw_names_list	gbif_names_list slw_names_list
NameStatusConditional	nameStatus	synpass_flags synfail_flags
CreateID	name	id_param
FirstElement_From_XPath	xpath xmltext	nodeVal
DC_Choose_Sub_Flow	internalCSVData	synExpOccRetCSVData dataSelCSVData dataQualCSVData endWFlowCSVData endWFlow csvData
gbifTaxonSearchParser	gbifTaxonSerachJSON	synTaxonIDList acceptedNameResponse rank
Concat_Response	accNameRes datasetName synRes datasetID	concatResponse
AssignInputOutput	in	out
GBIF_Data_Use_Conditional	copyright_answer	gbif_data_use_conditional
slw_filter_generator	sciName	filter
Col_Copyright_Conditional	copyright_answer	col_copyright_conditional
Parse_Project	jsonStr	jsonErr projectID percent
gbifNameSearchParser	gbifNameSerachJSON	taxonIDList emptyTaxonIDList
Split_GBIFChklist_Name_Id	gbifChkListNameId	gbifChkListName gbifChkListId
Empty_Response_Service	datasetName datasetID	emptyResponse
nameParser	taxonSearchJSON	synResponse
Parse_Data_Upload	jsonStr	jsonErr upload_ok
trimRESTurlResult	url	resultUrl
checkDataUpload	status	dataUpload_ok dataUpload_failed uploadStatus
SynCheckGUI	synreqres_list	names synreqres
DCSynExpInputParser	csvData	synonymRequest incorrectRecords
GBIFCheckListParser	gbifChkListJSON	gbifChkList
DCSynExpInputDialog	gbifChkLists synonymRequest	colSynReq gbifSynReq gbifSelChkListIDs colChosen gbifChosen
Syn_Credit_Checker	col_copyright_conditional colSynReq gbif_data_use_conditional gbifSynReq	colSynReq gbifSynReq
Merge_Syn_Responses	colSynResList gbifSynResList	synResList
Export_Data_Options	projectID	options
FirstElement_From_XPath	xpath xmltext	nodeVal

Outputs (3)

Name	Description
endWFlow
csv_output
file_write_ok

Datalinks (11)

Source	Sink
title_value:value	Select_File:title
Select_File:selectedFile	Read_Text_File:fileurl
Read_Text_File:filecontents	Data_Cleaning_Worklow_Loop:internalCSVData
no_separator:value	Merge_String_List_to_a_String:seperator
Data_Cleaning_Worklow_Loop:internalCSVData	Merge_String_List_to_a_String:stringlist
Select_Output_CSV_File:selectedFile	Write_Text_File:outputFile
Merge_String_List_to_a_String:concatenated	Write_Text_File:filecontents
output_filechooser_title:value	Select_Output_CSV_File:title
Data_Cleaning_Worklow_Loop:endWFlow	endWFlow
Merge_String_List_to_a_String:concatenated	csv_output
Write_Text_File:writeOK	file_write_ok

Coordinations (1)

Controller	Target
Merge_String_List_to_a_String	Select_Output_CSV_File

Information Workflow Type

Taverna 2

Information Uploader

Cherian Mathew

Information License

All versions of this Workflow are licensed under:

Information Version 5 (of 17)

View version:

Information Credits (1)

(People/Groups)

Cherian Mathew

Information Attributions (0)

(Workflows/Files)

None

Information Tags (21)

Uploader tags

Log in to add Tags

Information Shared with Groups (2)

Information Featured In Packs (1)

Private pack

Log in to add to one of your Packs

Information Attributed By (0)

(Workflows/Files)

None

Information Favourited By (1)

Pgrobe

Information Statistics

12160 viewings

2620 downloads

[ see breakdown ]

Citations (0)

None

Version History

In chronological order:

Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Wednesday 11 April 2012 10:08:25 (UTC)

Last edited by Cherian Mathew on Wednesday 11 April 2012 10:09:19 (UTC)
Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Wednesday 13 June 2012 10:31:04 (UTC)

Last edited by Cherian Mathew on Wednesday 13 June 2012 10:32:17 (UTC)

Revision comment:

Integrated new version of Synonym Expansion / Occurrence Retrieval part of the workflow
Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Tuesday 26 June 2012 12:47:53 (UTC)
Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Tuesday 26 June 2012 13:22:41 (UTC)

Last edited by Cherian Mathew on Thursday 28 June 2012 08:00:16 (UTC)

Revision comment:

Added new version of BioSTIF workflow which call the web service on http, removing the need for the credentials in Taverna.
Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Thursday 26 July 2012 14:50:56 (UTC)

Last edited by Cherian Mathew on Thursday 26 July 2012 14:54:16 (UTC)

Revision comment:

Added new version of taxonomic name expansion nested workflow
Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Tuesday 07 August 2012 08:14:45 (UTC)

Last edited by Cherian Mathew on Tuesday 07 August 2012 08:16:55 (UTC)
Taxonomic Data Refinement (Integrated) Workflow

Created by Cherian Mathew on Friday 28 September 2012 13:56:56 (UTC)

Last edited by Cherian Mathew on Friday 28 September 2012 13:57:17 (UTC)
Data Refinement Workflow

Created by Cherian Mathew on Thursday 01 November 2012 09:15:21 (UTC)
Data Refinement Workflow

Created by Cherian Mathew on Thursday 08 November 2012 13:12:58 (UTC)
Data Refinement Workflow

Created by Cherian Mathew on Thursday 31 January 2013 09:02:16 (UTC)
[PLUGIN]Data Refinement Workflow

Created by Cherian Mathew on Tuesday 02 April 2013 11:47:04 (UTC)

Revision comment:

This version of the workflow is compatible with the newly created DRF Plugin which installs the dependency jars using the plugin framework of taverna.
Data Refinement Workflow

Created by Cherian Mathew on Tuesday 02 April 2013 11:54:29 (UTC)

Revision comment:

changed title
Data Refinement Workflow v13

Created by Cherian Mathew on Thursday 01 August 2013 13:31:20 (UTC)

Revision comment:

Added checks for Google Refine.
Changed copyright notices to be popups.
Added BGBM EDIT Platform as a target.
Misc. look and feel changes
Data Refinement Workflow v14

Created by Cherian Mathew on Tuesday 04 March 2014 11:13:25 (UTC)

Revision comment:

This new version of the data refinement workflow has:
* Access to the new GBIF API for taxonomic name resolution and for retrieval of occurrence points.
* Integration of PESI name resolution web service.
* Added search filter for aggregated name checklists
* Minor changes in GUI, mainly in the TNRS subworkflow
* Minor bugs fixed
Data Refinement Workflow v15

Created by Biodiversity eLaboratory on Thursday 08 May 2014 12:52:48 (UTC)
Data Refinement Workflow v16

Created by Biodiversity eLaboratory on Tuesday 02 September 2014 08:35:48 (UTC)

Revision comment:

This new version of the data refinement workflow has:
* The incluision of the World Register of Marine Species (WoRMS) as a new checklist to resolve species names in the Taxonomic Name Resolution subworkflow. The WoRMS checklist is queried by using the WoRMS web service created by VLIZ.
* Minor bugs fixed
Data Refinement Workflow v17

Created by Biodiversity eLaboratory on Wednesday 17 December 2014 12:51:15 (UTC)

Revision comment:

Change the BioSTIF services from Fraunhofer to EGI