diff --git a/DataExplore.r b/DataExplore.r index 5da16e0..c593192 100644 --- a/DataExplore.r +++ b/DataExplore.r @@ -36,30 +36,45 @@ c("/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_374 -# Unwanted data (incomplete scanning, demo for tour, etc.) -# See the interim results from this script (DataExplore.r) -#------------------------------------------------------- -# EXP ID TREATMENT DFP SCAN_DATETIME_BY15MIN NUM_SCANS -#------------------------------------------------------- +# List unwanted data +# (created by incomplete scanning, demo during AAPF tour visits, etc.) +# +# How to specify unwanted measurements +# +# 1. Run this script (DataExplore.r). +# 2. In the output of DataExplore.r (Overview 1 & 2), +# you'll see a list of potential unwanted measurements +# (e.g., a handful of measurements in an enough time period). +# 3. Review this list and decide which measurements you want +# to exclude from your analysis. +# 4. Look for a variable named df.unwanted below. +# This variable is a list where you can specify the unwanted +# measurements you identified in step 3. +# 5. Fill in the list under df.unwanted with the attributes of the +# measurements you want to exclude. The number of items in the list will +# depend on your specific experiment and the output from DataExplore.r. +# +# Unwanted data example: +#---------------------------------------------------------- +# EXP ID TREATMENT DFP SCAN_DATETIME_BY15MIN NUM_SCANS +#---------------------------------------------------------- # 1 -#374 20 -1 59 2023-10-30 11:15:00 4 - +# 374 20 -1 59 2023-10-30 11:15:00 4 +# # 2 -#374 50-3 27 2023-09-28 09:15:00 2 - +# 374 50-3 27 2023-09-28 09:15:00 2 +# # 3 -#374 50-3 27 2023-09-28 09:00:00 24 -# or -#374 50-3 32 2023-10-03 15:00:00 24 - +# 374 50-3 27 2023-09-28 09:00:00 24 OR +# 374 50-3 32 2023-10-03 15:00:00 24 +# # 4 -#374 50-4 27 2023-09-28 09:45:00 24 -# or -#374 50-4 32 2023-10-03 15:00:00 24 - +# 374 50-4 27 2023-09-28 09:45:00 24 OR +# 374 50-4 32 2023-10-03 15:00:00 24 +# # 5 -#396 20-1 55 2024-04-29 12:00:00 1 -#------------------------------------------------------- +# 396 20-1 55 2024-04-29 12:00:00 1 +#---------------------------------------------------------- df.unwanted <- data.frame( EXP_ID=c(374, 374, 374, 374, 396), TREATMENT=c("20 -1", "50-3", "50-3", "50-4", "20-1"), @@ -71,7 +86,17 @@ df.unwanted <- data.frame( stringsAsFactors=FALSE) -# Treatment name change +# List descriptive treatment info +# +# The df.convert.treatment placeholder allows you to change +# treatment attributes assigned by PPEW. It takes a list of pairs, +# where each pair contains: +# +# - Input Treatment Attribute (from): check from Overview 1 & 2. +# - New treatment attribute (to): a string (space allowed) that +# best describes the treatment itself. +# +# Treatment change info example: df.convert.treatment <- data.frame( from=c("20 -1", "20-2", "50-3", "50-4", "20-1", "20-2"), to =c("20", "20", "80", "80", "20", "20"), @@ -79,6 +104,93 @@ df.convert.treatment <- data.frame( +# List growth stage info +# +# The df.growth.stage variable lets you define the growth stage +# for each data point based on a combination of three factors: +# EXP ID, TREATMENT, DFP +# +# To determine the appropriate growth stage for each data point +# based on this combination, refer to Overview 5. +# +# Growth stage info Example: +df.growth.stage <- data.frame( + EXP_ID =c(374, 374, 374, 374, + 375, 375, 375, 375, + 378, 378, 378, 378, + 380, 380, 380, 380, + 390, 390, 390, 390, + 391, 391, 391, 391, + 396, 396, 396, 396, + 401, 401, 401, 401), + TREATMENT =c("20", "20", "80", "80", + "20", "20", "80", "80", + "20", "20", "80", "80", + "20", "20", "80", "80", + "20", "20", "80", "80", + "20", "20", "80", "80", + "20", "20", "80", "80", + "20", "20", "80", "80"), + DFP =c(33, 59, 32, 50, + 35, 56, 28, 48, + 35, 60, 28, 56, + 34, 63, 28, 53, + 35, 63, 28, 48, + 35, 63, 28, 56, + 42, 70, 42, 63, + 42, 73, 43, 65), + GROWTH_STAGE=c('V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT')) + +df.growth.stage <- data.frame( + EXP_ID =c(374, 374, 374, 374, + 375, 375, 375, 375, + 378, 378, 378, 378, + 380, 380, 380, 380, + 390, 390, 390, 390, + 391, 391, 391, 391, + 396, 396, 396, 396, + 401, 401, 401, 401), + TREATMENT =c(20, 20, 80, 80, + 20, 20, 80, 80, + 20, 20, 80, 80, + 20, 20, 80, 80, + 20, 20, 80, 80, + 20, 20, 80, 80, + 20, 20, 80, 80, + 20, 20, 80, 80), + DFP =c(33, 59, 32, 50, + 35, 56, 28, 48, + 35, 60, 28, 56, + 34, 63, 28, 53, + 35, 63, 28, 48, + 35, 63, 28, 56, + 42, 70, 42, 63, + 42, 73, 43, 65), + GROWTH_STAGE=c('V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT', + 'V8', 'VT', 'V8', 'VT')) + + + + + + + + + + @@ -223,7 +335,7 @@ message('Succeeded importing HSI data...') df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] # Show summary of rgb data -message('\nRGB Data Acquisiton Summary\n') +message('\nRGB Data Acquisiton Summary (Overview 1)\n') print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) # Round down scan date and time down to nearest 15 minute and display @@ -233,11 +345,11 @@ df.rgb.perscan["SCAN_DATETIME"] = format=format) df.rgb.perscan["SCAN_DATETIME_BY15MIN"] = lubridate::round_date(df.rgb.perscan$SCAN_DATETIME, "15 minutes") -message('\nRGB Data Acquisiton Summary by Date and Time\n') +message('\nRGB Data Acquisiton Summary by Date and Time (Overview 2)\n') print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP, SCAN_DATETIME_BY15MIN)) -message('\nPlease use [`EXP ID`, TREATMENT, SCAN_DATETIME_BY15MIN]') +message('\nPlease check [`EXP ID`, TREATMENT, SCAN_DATETIME_BY15MIN]') message('to exclude data from incomplete scan session...') -message('Check df.unwanted in DataExplore.r to check how to exclude records...\n') +message('Check df.unwanted in DataExplore.r to exclude such records...\n') @@ -253,9 +365,8 @@ for(i in 1:nrow(df.unwanted)){ TREATMENT == row$TREATMENT & SCAN_DATETIME_BY15MIN == row$SCAN_DATETIME_BY15MIN)) } -# Show data acquisiton summary of filtering out unwanted data df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] -message('\nRGB Data Acquisiton Summary (Unwanted Data Filtered)\n') +message('\nRGB Data Acquisiton Summary (Unwanted Data Filtered, Overview 3)\n') print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) @@ -267,9 +378,8 @@ for(i in 1:nrow(df.convert.treatment)){ row <- df.convert.treatment[i,] df.rgb$TREATMENT[df.rgb$TREATMENT==row$from] <- row$to } -# Show data acquisiton summary of filtering out unwanted data df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] -message('\nRGB Data Acquisiton Summary (Treatment Info Updated)\n') +message('\nRGB Data Acquisiton Summary (Treatment Updated, Overview 4)\n') print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) @@ -278,23 +388,45 @@ print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) # DFP is always smaller than 365 (to prevent case with wrong year input) df.rgb$DFP = df.rgb$DFP %% 365 -# Show data acquisiton summary of filtering out unwanted data df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] -message('\nRGB Data Acquisiton Summary (DFP is always lower than 365)\n') +message('\nRGB Data Acquisiton Summary (DFP Always Less Than 365, Overview 5)\n') print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) -# Set Growth stage +# Add growth stage attribute +df.rgb$GROWTH_STAGE <- NA +for(i in 1:nrow(df.growth.stage)){ + row <- df.growth.stage[i,] + df.rgb$GROWTH_STAGE[(df.rgb$`EXP ID` == row$EXP_ID) & + (df.rgb$TREATMENT == row$TREATMENT) & + (df.rgb$DFP == row$DFP)] <- row$GROWTH_STAGE +} +df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] +message('\nRGB Data Acquisiton Summary (Growth Stage Added, Overview 6)\n') +print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP, GROWTH_STAGE)) + + + + + + + +# Remove SCAN_TIME SCAN_DATE SCAN_DATETIME SCAN_DATETIME_BY15MIN + +# Save data? + +# Repeat for df.hsi + +######### Combine rgb and hsi data using df <- rbind(df.rgb, df.hsi) + + + # frame_nr 0-11 (normal) , 0-23 for one scan # For "RGB-SideAll", change column names Frame0-11 to major, major+30... -# Remove SCAN_TIME SCAN_DATE SCAN_DATETIME SCAN_DATETIME_BY15MIN -# Do the same for df.hsi -######### Combine rgb and hsi data -#########df <- rbind(df.rgb, df.hsi)