From 007f882f53c558a07d39ced97473531b0e5842a1 Mon Sep 17 00:00:00 2001 From: Sungchan Oh Date: Tue, 2 Jul 2024 16:27:44 -0400 Subject: [PATCH] Load rgb data (working) --- DataExplore.r | 168 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 156 insertions(+), 12 deletions(-) diff --git a/DataExplore.r b/DataExplore.r index fe80e72..5da16e0 100644 --- a/DataExplore.r +++ b/DataExplore.r @@ -5,11 +5,78 @@ library(dplyr) library(readxl) library(data.table) +library(lubridate) # File path to masterfiles -paths.rgb <- c("../RGB_Bayer_2.xlsx") -paths.hsi <- c("../HS_Bayer_2.xlsx") +#paths.rgb <- c("../RGB_Bayer_2.xlsx") +#paths.hsi <- c("../HS_Bayer_2.xlsx") + + +# File path to the individual masterfiles for each AAPF experiment +paths.rgb <- +c("/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_374.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_375.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_378.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_380.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_390.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_391.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_396.xlsx", +"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_401.xlsx") + +paths.hsi <- +c("/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_374.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_375.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_378.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_380.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_390.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_391.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_396.xlsx", +"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_401.xlsx") + + + +# Unwanted data (incomplete scanning, demo for tour, etc.) +# See the interim results from this script (DataExplore.r) +#------------------------------------------------------- +# EXP ID TREATMENT DFP SCAN_DATETIME_BY15MIN NUM_SCANS +#------------------------------------------------------- +# 1 +#374 20 -1 59 2023-10-30 11:15:00 4 + +# 2 +#374 50-3 27 2023-09-28 09:15:00 2 + +# 3 +#374 50-3 27 2023-09-28 09:00:00 24 +# or +#374 50-3 32 2023-10-03 15:00:00 24 + +# 4 +#374 50-4 27 2023-09-28 09:45:00 24 +# or +#374 50-4 32 2023-10-03 15:00:00 24 + +# 5 +#396 20-1 55 2024-04-29 12:00:00 1 +#------------------------------------------------------- +df.unwanted <- data.frame( + EXP_ID=c(374, 374, 374, 374, 396), + TREATMENT=c("20 -1", "50-3", "50-3", "50-4", "20-1"), + SCAN_DATETIME_BY15MIN=c("2023-10-30 11:15:00", + "2023-09-28 09:15:00", + "2023-09-28 09:00:00", + "2023-09-28 09:45:00", + "2024-04-29 12:00:00"), + stringsAsFactors=FALSE) + + +# Treatment name change +df.convert.treatment <- data.frame( + from=c("20 -1", "20-2", "50-3", "50-4", "20-1", "20-2"), + to =c("20", "20", "80", "80", "20", "20"), + stringAsFactors=FALSE) + @@ -30,14 +97,13 @@ regex.col.hsi <- paste("EXP ID", "POT_BARCODE", "VARIETY", "TREATMENT", sep="|") # Column names to be disregarded -nouse.col.rgb <- c("Filename", "SCAN_TIME", "SCAN_DATE") +nouse.col.rgb <- c("Filename") nouse.col.hsi <- c("Filename-VNIR-SIDE", "Filename-VNIR-TOP", - "Filename-SWIR-SIDE", "Filename-SWIR-TOP", - "SCAN_TIME", "SCAN_DATE") + "Filename-SWIR-SIDE", "Filename-SWIR-TOP") -# Column names used as identifier -id.vars <- c("EXP ID", "POT_BARCODE", "VARIETY", - "TREATMENT", "DFP", "View", "frame_nr") +# Column names used as interim identifier +id.vars <- c("EXP ID", "POT_BARCODE", "SCAN_TIME", "SCAN_DATE", + "VARIETY", "TREATMENT", "DFP", "View", "frame_nr") @@ -47,9 +113,11 @@ id.vars <- c("EXP ID", "POT_BARCODE", "VARIETY", mat <- matrix(ncol=0, nrow=0) df.rgb <- data.frame(mat) +# Import rgb data for (path.rgb in paths.rgb){ for (tab.rgb in excel_sheets(path = path.rgb)){ if (tab.rgb=="PPEW") next + if (tab.rgb=="Information") next # Read a rgb worksheet temp <- read_excel(path.rgb, sheet=tab.rgb) @@ -71,7 +139,6 @@ for (path.rgb in paths.rgb){ # Assign values in the id columns that didn't exist for (col.to.add in cols.to.add){ - # TODO if (col.to.add == "View" && tolower(tab.rgb) =="top"){ temp["View"] = "Top" } @@ -98,13 +165,17 @@ message('Succeeded importing RGB data...') -# Create empty rgb dataframe + + +# Create empty hsi dataframe mat <- matrix(ncol=0, nrow=0) df.hsi <- data.frame(mat) +# Import hsi data for (path.hsi in paths.hsi){ for (tab.hsi in excel_sheets(path = path.hsi)){ if (tab.hsi=="PPEW") next + if (tab.hsi=="Information") next # Read a hsi worksheet temp <- read_excel(path.hsi, sheet=tab.hsi) @@ -148,15 +219,88 @@ message('Succeeded importing HSI data...') +# Get unique rgb measurement +df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] + +# Show summary of rgb data +message('\nRGB Data Acquisiton Summary\n') +print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) + +# Round down scan date and time down to nearest 15 minute and display +format <- "%Y-%m-%d %H:%M:%S" +df.rgb.perscan["SCAN_DATETIME"] = + as.POSIXct(paste(as.Date(df.rgb.perscan$SCAN_DATE), df.rgb.perscan$SCAN_TIME), + format=format) +df.rgb.perscan["SCAN_DATETIME_BY15MIN"] = + lubridate::round_date(df.rgb.perscan$SCAN_DATETIME, "15 minutes") +message('\nRGB Data Acquisiton Summary by Date and Time\n') +print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP, SCAN_DATETIME_BY15MIN)) +message('\nPlease use [`EXP ID`, TREATMENT, SCAN_DATETIME_BY15MIN]') +message('to exclude data from incomplete scan session...') +message('Check df.unwanted in DataExplore.r to check how to exclude records...\n') + + + + +# Drop unwanted rgb data +df.rgb["SCAN_DATETIME"] = + as.POSIXct(paste(as.Date(df.rgb$SCAN_DATE), df.rgb$SCAN_TIME), format=format) +df.rgb["SCAN_DATETIME_BY15MIN"] = + lubridate::round_date(df.rgb$SCAN_DATETIME, "15 minutes") +for(i in 1:nrow(df.unwanted)){ + row <- df.unwanted[i,] + df.rgb <- df.rgb %>% filter(!(`EXP ID` == row$EXP_ID & + TREATMENT == row$TREATMENT & + SCAN_DATETIME_BY15MIN == row$SCAN_DATETIME_BY15MIN)) +} +# Show data acquisiton summary of filtering out unwanted data +df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] +message('\nRGB Data Acquisiton Summary (Unwanted Data Filtered)\n') +print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) + + + + + +# Convert treatment attributes +for(i in 1:nrow(df.convert.treatment)){ + row <- df.convert.treatment[i,] + df.rgb$TREATMENT[df.rgb$TREATMENT==row$from] <- row$to +} +# Show data acquisiton summary of filtering out unwanted data +df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] +message('\nRGB Data Acquisiton Summary (Treatment Info Updated)\n') +print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) + -# Merge RGB and HSI data +# DFP is always smaller than 365 (to prevent case with wrong year input) +df.rgb$DFP = df.rgb$DFP %% 365 +# Show data acquisiton summary of filtering out unwanted data +df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ] +message('\nRGB Data Acquisiton Summary (DFP is always lower than 365)\n') +print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP)) + + + + +# Set Growth stage +# frame_nr 0-11 (normal) , 0-23 for one scan # For "RGB-SideAll", change column names Frame0-11 to major, major+30... +# Remove SCAN_TIME SCAN_DATE SCAN_DATETIME SCAN_DATETIME_BY15MIN + +# Do the same for df.hsi +######### Combine rgb and hsi data +#########df <- rbind(df.rgb, df.hsi) + + + + + -# Change DFPs to growth stage