Changeset Navigation

← Previous Changeset
Next Changeset →

Changeset 235

Timestamp:

09/17/08 13:06:50

Author:

dan

Message:

Files:

scout/trunk/GetLatestData.pl (modified) (13 diffs)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

scout/trunk/GetLatestData.pl

r216	r235
107	107	#my $qs_only = @ARGV[0];
108	108
109		#12/5/05 Payne Seal changed all http://nccoos.unc.edu to http://nemo.isis.unc.edu per Sara Haines
110		#April 9, 2008 Jeremy Cothran changed all http://nemo.isis.unc.edu to http://whewell.marine.unc.edu per Jesse Cleary
111
112		=comment
113		if (length($qs_only) <= 0) {
114		@dir_urls = (
115		##'http://nccoos.unc.edu/data/nos/proc_data/latest_v2.0'
116		#'http://carocoops.org/~mkanoth/seacoos_netcdf/'
117		#,'http://whewell.marine.unc.edu/data/nos/proc_data/latest_v2.0/'
118		'http://seacoos.skio.peachnet.edu/proc_data/latest_v2.0'
119		##,'http://nccoos.unc.edu/data/nws_metar/proc_data/latest_v2.0'
120		,'http://whewell.marine.unc.edu/data/nws_metar/proc_data/latest_v2.0/'
121		#,'http://trident.baruch.sc.edu/usgs_data'
122		#,'http://trident.baruch.sc.edu/storm_surge_data/latest'
123		#,'http://trident.baruch.sc.edu/storm_surge_data/netcdf_latest'
124		,'http://seacoos.marine.usf.edu/data/seacoos_rt_v2'
125		#,'http://oceanlab.rsmas.miami.edu/ELWX5/v2'
126		#,'http://aigeann.ucsc.edu/cimtLatest'
127		##,'http://nccoos.unc.edu/data/nc-coos/latest_v2.0'
128		#,'http://nemo.isis.unc.edu/data/nc-coos/latest_v2.0/' #nccoos hf radar
129		##,'http://iwave.rsmas.miami.edu/wera/efs/data'
130		#,'http://oceanlab.rsmas.miami.edu/wera' #miami hf radar
131		##,'http://ak.aoos.org/seacoos/codar'
132		,'http://www.cormp.org/data'
133		##'http://www.cormp.org/data'
134		);
135		}
136		else {
137		@dir_urls = (
138		#'http://trident.baruch.sc.edu/po.daac_data/OVW'
139		);
140		$qs_prefix = 'qs_';
141		}
142		=cut
	109
143	110
144	111	#DWR v1.1.0.0
…	…
148	115	{
149	116	my $cmd = "rm /home/dramage/log/cdl_0master.log";
150		print( "Remove Log: $cmd\n" );
151		`$cmd`;
	117	# print( "Remove Log: $cmd\n" );
	118	# `$cmd`;
152	119	#Open up a new log file.
153		$cmd = "echo cdl_0master log opened > /home/dramage/log/cdl_0master.log";
	120	$cmd = "echo cdl_0master log opened >> /home/dramage/log/cdl_0master.log";
154	121	`$cmd`;
155	122	}
…	…
163	130	{
164	131	my $this_dir_url = $URL->string_value();;
165
166		print ~~"$this_dir_url\n"~~;
	132	print('-------------------------------------------------------------------------------------------------'."\n" );
	133	print( "NetCDF Source URL: $this_dir_url\n" );
167	134	my @latest = get_index_match( $this_dir_url, $strFilter );
168	135	foreach (@latest)
…	…
170	137	my $latest_file = $_;
171	138	my $strLatestFile;
172		if( !MICROSOFT_PLATFORM )
173		{
174		$strLatestFile = "$this_dir_url/$latest_file"
175		}
176		else
177		{
178		$strLatestFile = "$this_dir_url\\$latest_file"
179		}
	139
	140	$strLatestFile = "$this_dir_url/$latest_file";
	141
180	142	my ($content_type, $document_length, $modified_time, $expires, $server) = head($strLatestFile);
181	143
182		print ~~" $latest_file\n "~~;
	144	print( "File to check against cache: $latest_file\n" );
183	145	my $latest_file_filter = $_;
184	146	my $latest_file_filter =~ s/-/_/g;
…	…
187	149	my $strLogFile;
188	150	$strSourceFile = "$this_dir_url/$latest_file";
	151
	152	#DWR v1.1.1.0
	153	#Find the file extension.
	154	my $iPos = rindex( $latest_file, '.' );
	155	my $strFileName = $latest_file;
	156	if( $iPos != -1 )
	157	{
	158	$strFileName = substr( $latest_file, 0, $iPos );
	159	}
189	160	if( !MICROSOFT_PLATFORM )
190	161	{
	162	#my $strDate = `date +%Y%m%d%H%M%S`;
	163	#chomp( $strDate );
	164	#$strFileName = $strFileName."-$strDate.nc";
	165	#$strDestFile = "$strNetCDFDir/$strFileName";
	166
191	167	$strDestFile = "$strNetCDFDir/$latest_file";
192		$strLogFile = "$strFetchLogDir/$latest_file";
	168	$strLogFile = "$strFetchLogDir//$latest_file";
193	169	}
194	170	else
195	171	{
	172	#my $strDate = `\\UnixUtils\\usr\\local\\wbin\\date.exe +%Y%m%d%H%M%S`;
	173	#chomp( $strDate );
	174	#$strFileName = $strFileName."-$strDate.nc";
	175	#$strDestFile = "$strNetCDFDir\\$strFileName";
	176
196	177	$strDestFile = "$strNetCDFDir\\$latest_file";
197	178	$strLogFile = "$strFetchLogDir\\$latest_file";
198	179	}
199		print( "strDestFile = $strDestFile strLogFile = $strLogFile\n" );
	180	#print( "strDestFile = $strDestFile strLogFile = $strLogFile\n" );
200	181
201	182	my $bDownloadFile = 0;
…	…
203	184	{
204	185	my $last_fetch_time = <LAST_FETCH>;
205		print ~~'file last modified on record = '.scalar localtime($last_fetch_time)."\n "~~;
	186	print( "\t".'Fetch log file: '.$strLogFile.' last modified on record = '.scalar localtime($last_fetch_time)."\n" );
206	187	chop($last_fetch_time);
207	188	if ($modified_time > $last_fetch_time)
…	…
212	193	else
213	194	{
	195	print( "\tUnable to open Fetch log file: $strLogFile.\n" );
214	196	$bDownloadFile = 1;
215	197	}
…	…
219	201	if( is_success($RetVal) )
220	202	{
221		print ~~"Downloaded: $strSourceFile to $strDestFile"~~;
	203	print( "\tDownloaded: $strSourceFile to $strDestFile\n" );
222	204	if( $bWriteObsKML )
223	205	{
…	…
227	209	else
228	210	{
229		print "ERROR: Fail Download: $strSourceFile to $strDestFile, Return Code: $RetVal";
230		}
231		}
232		print "\n ".'file last modified = '.scalar localtime($modified_time)."\n";
	211	print( "ERROR: Fail Download: $strSourceFile to $strDestFile, Return Code: $RetVal\n" );
	212	}
	213	}
	214	else
	215	{
	216	print( "\tNot downloaded.\n" );
	217	}
	218	print( "\t".'File last modified = '.scalar localtime($modified_time)."\n" );
233	219	my $cmd;
234	220	if( !MICROSOFT_PLATFORM )
…	…
240	226	$cmd = "\\UnixUtils\\usr\\local\\wbin\\touch $strFetchLogDir/$latest_file";
241	227	}
242		~~print( "~~Touch: $cmd\n" );
	228	#print( "\tTouch: $cmd\n" );
243	229	`$cmd`;
244	230	if( !MICROSOFT_PLATFORM )
…	…
251	237	}
252	238	`$cmd`;
253		if( $b~~ProcessNetCDF~~ )
	239	if( $bDownloadFile )
254	240	{
255	241	if( !MICROSOFT_PLATFORM )
…	…
258	244	#Added the passing of $strLastNTimeStamps onto the command line.
259	245	$cmd = "cd /home/dramage/netcdf; /usr/bin/perl /home/dramage/netcdf/cdl_0master.pl $strDestFile 1 $strDirForObsKml $strLastNTimeStamps >> /home/dramage/log/cdl_0master.log 2>&1";
260		print( "ObsKML Script: $cmd\n" );
	246	print( "\tObsKML Script: $cmd\n" );
261	247	`$cmd`;
262	248	}
…	…
269	255	}
270	256	if( $bProcessNetCDF )
271		{
	257	{
	258	#`cd /home/dramage; /home/dramage/netcdf/DirZip.sh $strDirForObsKml kml`
	259	my $strDirtoList;
	260	my @aFeedDirs;
272	261	if( !MICROSOFT_PLATFORM )
273		{
274		`cd /home/dramage; /home/dramage/DirZip.sh $strDirForObsKml kml >> GetLatestData.log 2>&1`
275		}
	262	{
	263	$strDirtoList = $strDirForObsKml."/*";
	264	@aFeedDirs = `ls -dq $strDirtoList`;
	265
	266	#`cd /home/dramage; /home/dramage/netcdf/DirZip.sh $strDirForObsKml kml`
	267	}
	268	else
	269	{
	270	$strDirtoList = $strDirForObsKml."\\*";
	271	@aFeedDirs = `\\UnixUtils\\usr\\local\\wbin\\ls -dq $strDirtoList`;
	272	}
	273	print( "\nNumber of directories to zip: ". @aFeedDirs . "\n" );
	274	foreach my $strDir (@aFeedDirs)
	275	{
	276	chomp( $strDir );
	277
	278	my $iPos;
	279	if( !MICROSOFT_PLATFORM )
	280	{
	281	#Get rid of KML files older than 4 hours.
	282	#print( "\tfind $strDir -maxdepth 1 -cmin +240 -exec rm -f {}\n");
	283	#`find $strDir -maxdepth 1 -cmin +240 -exec rm -f {} \;`;
	284	$iPos = rindex( $strDir, '/' );
	285	my $strDirName = $strDir;
	286	if( $iPos != -1 )
	287	{
	288	$strDirName = substr( $strDir, $iPos+1, length( $strDir ) );
	289	}
	290	my $strZipName = $strDir.'/'.$strDirName.'_latest_obskml.zip';
	291	print( "\tRemoving previous zip: $strZipName.\n");
	292	`rm $strZipName`;
	293	#print( "\tZipping: zip -j $strZipName $strDir/*.kml\n" );
	294	print( "\tcd $strDir ; find -iname \'*.kml\' \| zip -j $strZipName -@\n" );
	295	`cd $strDir ; find -iname '*.kml' \| zip -j $strZipName -@`;
	296
	297	#`zip -j $strZipName $strDir/*.kml`;
	298	}
	299	else
	300	{
	301	#Get rid of KML files older than 4 hours.
	302	#print( "\tfind $strDir -maxdepth 1 -cmin +240 -exec rm -f {}\n");
	303	#`\\UnixUtils\\usr\\local\\wbin\\find $strDir -maxdepth 1 -cmin +240 -exec rm -f {} \;`;
	304	$iPos = rindex( $strDir, '\\' );
	305	my $strDirName = $strDir;
	306	if( $iPos != -1 )
	307	{
	308	$strDirName = substr( $strDir, $iPos+1, length( $strDir ) );
	309	}
	310	my $strZipName = $strDir.'\\'.$strDirName.'_latest_obskml.zip';
	311	print( "\tRemoving previous zip: $strZipName.\n");
	312	`\\UnixUtils\\usr\\local\\wbin\\rm $strZipName`;
	313	print( "\tcd $strDir ; \\UnixUtils\\usr\\local\\wbin\\find -iname \'*.kml\' \| zip -j $strZipName -@\n" );
	314	#print( "\tZipping: zip -j $strZipName $strDir\\*.kml\n" );
	315	`cd $strDir ; \\UnixUtils\\usr\\local\\wbin\\find -iname '*.kml' \| zip -j $strZipName -@`;
	316	#`\\UnixUtils\\usr\\local\\wbin\\zip -j $strZipName $strDir\\*.kml`;
	317	}
	318	}
276	319	}
277	320

Changeset Navigation

Changeset 235

Legend:

scout/trunk/GetLatestData.pl

Download in other formats: