Changeset 201

Timestamp:

06/03/08 15:48:51

Author:

dan

Message:

Added the ability to select last N times from the netcdf files when generating obskml files.
Does not apply currently to the construction of the SQL files.

Files:

• scout/trunk/GetLatestData.pl (modified) (8 diffs)
• scout/trunk/cdl_0master.pl (modified) (7 diffs)
• scout/trunk/cdl_fixed_point.pl (modified) (16 diffs)

scout/trunk/GetLatestData.pl

@@ -9 +9 @@
-
-#Get the minute so we know when we are at the top of the hour, we can delete the last hours .zip file.
+=comment
-my $Minute;
-if( !MICROSOFT_PLATFORM )
@@ -18 +19 @@
-  $Minute = `\\UnixUtils\\usr\\local\\wbin\\date %M`;
-}
-
+
-my %CommandLineOptions;
-GetOptions( \%CommandLineOptions,
@@ -26 +27 @@
-            "Delete:s",             #Optional. If we are converting to obsKML and want to get rid of the netcdfs afterwards, set this to yes.
-            "DownloadDir:s",        #Optional. Directory to store the downloaded netcdf files in. Default is ./latest_netcdf_files.
-
+
+
-             );
-
-my $strURLControlFile  = $CommandLineOptions{"URLControlFile"}; 
- 
+
-my $strDirForObsKml    = $CommandLineOptions{"DirForObsKml"}; 
-my $strDelete          = $CommandLineOptions{"DeleteFiles"}; 
-my $strNetCDFDir       = $CommandLineOptions{"NetCDFDir"};
-my $strFetchLogDir     = $CommandLineOptions{"FetchLogDir"};
+my $strLastNTimeStamps   = $CommandLineOptions{"UseLastNTimeStamps"}; #DWR v1.1.0.0 6/2/2008
-if( length( $strURLControlFile ) == 0 )
-{
@@ -44 +47 @@
-              "--Delete specifies we delete the netcdf files after we write the ObsKML files. This is an optional argument.\n".
-              "--DownloadDir is the directory to store the downloaded files. Optional, default is ./latest_netcdf_files.\n".
-
+
+
-}
-if( length( $strNetCDFDir) == 0 )
@@ -86 +90 @@
-}
-
-print( "Command Line Options: strURLControlFile = $strURLControlFile strFilter = $strFilter strDirForObsKml = $strDirForObsKml strDelete = $strDelete strNetCDFDir = $strNetCDFDir strFetchLogDir = $strFetchLogDir\n" );
-
-my $bWriteObsKML = 0;
-if( length( $strDirForObsKml ) != 0 )
@@ -93 +95 @@
-  $bWriteObsKML = 1;
-}
+
+#DWR v1.1.0.0 6/2/2008
+if( length( $strLastNTimeStamps ) == 0 )
+{
+  $strLastNTimeStamps = 0;
+}
+
+print( "Command Line Options: strURLControlFile = $strURLControlFile strFilter = $strFilter strDirForObsKml = $strDirForObsKml strDelete = $strDelete strNetCDFDir = $strNetCDFDir strFetchLogDir = $strFetchLogDir UseLastNTimeStamps = $strLastNTimeStamps\n" );
+
-# qs needs its own individual cycle
-#my $qs_only = @ARGV[0];
@@ -131 +142 @@
-=cut
-
+#DWR v1.1.0.0
+#Remove the conversion log. We do this since the log gets concatenated every run, so it will get huge fast.
+#We just keep the log of the last run.
+if( !MICROSOFT_PLATFORM )
+{
+  my $cmd = "rm /home/dramage/log/cdl_0master.log";
+  print( "Remove Log: $cmd\n" );
+  `$cmd`;      
+  #Open up a new log file.
+  $cmd = "echo cdl_0master log opened > /home/dramage/log/cdl_0master.log";
+  `$cmd`;
+}
+
-my $bProcessNetCDF = 0;
-my $XMLObj = XML::LibXML->new->parse_file($strURLControlFile);
@@ -231 +255 @@
-        if( !MICROSOFT_PLATFORM )
-        {
-
+
+
+
-          print( "ObsKML Script: $cmd\n" );
-          `$cmd`;      

scout/trunk/cdl_0master.pl

@@ -1 +1 @@
-###############################################################################
-# Revisions
+# REv: 1.1.0.0
+# Author: DWR
+# Date: 6/2/2008
+# Changes: Optional. Added command line argument LastNTimeStamps to allow the use of only the last N time stamps
+# of the data.
-# Author: DWR
-# Date: 4/18/2008
@@ -30 +35 @@
-use obsKMLSubRoutines;
-
+use constant USE_DEBUG_PRINTS => 1;
+
-#DWR 4/7/2008 Added constants for use with ARGV[1] to tell us what file(s) to write.
-use constant {
@@ -68 +75 @@
-my $FileCreationOption = 0;
-my $strObsKMLFilePath;
+my $iLastNTimeStamps = 0;
-if( @ARGV == 0 )
-{
@@ -73 +81 @@
-       "Parameter 0 is the netcdf filename to process.\n".
-       "Parameter 1 is an optional parameter. Valid values are: 0 Write only SQL files, 1 Write only obsKML file, or 2 Write both SQL files and obsKML file. Default value is 0.\n".
-
+
+
+
-}
-if( @ARGV > 1 )
@@ -91 +101 @@
-    $strObsKMLFilePath = @ARGV[2];
-  }
+  if( @ARGV >= 4 )
+  {
+    $iLastNTimeStamps = @ARGV[3];
+  }
-}
+print( "Processing: $net_cdf_file---------------------------------------------------------------------------------------------------------\n" );
+print( "Command Line Args: net_cdf_file: $net_cdf_file FileType=$FileCreationOption ObsKMLPath=$strObsKMLFilePath LastNTimes=$iLastNTimeStamps\n" );
-#
-# Turn off fatal error aborts.  We need to allow some attributes to error out w/o crashing.
@@ -211 +227 @@
-  } 
-  # Added parameters $FileCreationOption and $strObsKMLFilePath.
-
+
+
+
+
+
-}
-elsif ($format_category_value eq 'moving-point-2D') {
@@ -225 +245 @@
-  fixed_map();
-}
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

scout/trunk/cdl_fixed_point.pl

@@ -1 @@
-
-
-
+
-
-  # top timestamp
@@ -8 +6 @@
-  my $FileCreationOptions = $_[1];
-  my $strObsKMLFilePath = $_[2];
-
+
+
+
+
-  my $bWriteSQLFiles = 1;
-  my $bWriteobsKMLFile = 0;
@@ -388 +389 @@
-  my $varget = NetCDF::varget($ncid, $time_var{'var_id'}, (0), $time_dim{'dim_size'},   \@time_values);
-  if ($varget < 0) {die "ABORT! Cannot get time values.\n";}
+  #DWR v1.1.0.0
+  #If we using the last N times, let's populate the array.
+   my $iStartingNdx = 0;
+  if( $iLastNTimeStamps > 0 )
+  {
+    #If we have more entries in the time_values array, let's set the starting index at the spot in the array which   
+    #will get us to the first of the N time stamps.
+    my $iTimeCnt = @time_values;
+    if( $iTimeCnt > $iLastNTimeStamps )
+    {
+      $iStartingNdx = $iTimeCnt - $iLastNTimeStamps;
+      if( USE_DEBUG_PRINTS )
+      {
+        print( "fixed_point()::Printing Last: $iLastNTimeStamps out of $iTimeCnt entries. Starting Index: $iStartingNdx. $ncid\n" );
+      }
+    }
+  }
-  # get the units
-  my $attget = NetCDF::attget($ncid, $time_var{'var_id'}, 'units', \$units_value);
@@ -418 +436 @@
-      .sprintf("%02d",$base_minute).':'
-      .sprintf("%02d",$base_second);
+
-  }
-  
@@ -462 +481 @@
-      #DWR 4/5/2008
-      my $units_value = '';
-
+'units', 
-      if ($attget < 0) 
-      {
@@ -468 +487 @@
-      }
-      $water_level[$i]{'units'} = $units_value;
-      
-      # we need to loop through the attributes, so find out how many there are
-      my $varinq = NetCDF::varinq($ncid, $water_level[$i]{'var_id'},
@@ -1450 +1468 @@
-      my $NavD88DataVal = 'NULL';
-      my $Height = '';
-    
-      for my $i (0..$#water_level) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -1615 +1633 @@
-      my $DataVal = 'NULL';
-      my $Height = '';
+      
-      for my $i (0..$#sea_surface_temperature) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -1741 +1761 @@
-      for my $i (0..$#sea_bottom_temperature) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -1852 +1873 @@
-      for my $i (0..$#air_temperature) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -2011 +2033 @@
-        while ($j <= $#wind_from_direction
-          && $wind_from_direction[$j]{'height'} != $wind_speed[$i]{'height'}) 
-  
+
-          $j++;
-        }
@@ -2020 +2042 @@
-        else 
-        {
-
+
+
-          {
-            $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -2184 +2207 @@
-      for my $i (0..$#air_pressure) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -2295 +2319 @@
-      for my $i (0..$#salinity) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -2441 +2466 @@
-          else 
-          {
-
+
+
+
-            {
-              $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;
@@ -2630 +2657 @@
-      for my $i (0..$#significant_wave_height) 
-      {
-
+
+
-        {
-          $this_station_id = $institution_code_value.'_'.$platform_code_value.'_'.$package_code_value;