Skip to content

Commit

Permalink
Load rgb data (working)
Browse files Browse the repository at this point in the history
  • Loading branch information
Sungchan Oh committed Jul 2, 2024
1 parent 7cd3ca9 commit 007f882
Showing 1 changed file with 156 additions and 12 deletions.
168 changes: 156 additions & 12 deletions DataExplore.r
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,78 @@
library(dplyr)
library(readxl)
library(data.table)
library(lubridate)


# File path to masterfiles
paths.rgb <- c("../RGB_Bayer_2.xlsx")
paths.hsi <- c("../HS_Bayer_2.xlsx")
#paths.rgb <- c("../RGB_Bayer_2.xlsx")
#paths.hsi <- c("../HS_Bayer_2.xlsx")


# File path to the individual masterfiles for each AAPF experiment
paths.rgb <-
c("/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_374.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_375.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_378.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_380.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_390.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_391.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_396.xlsx",
"/depot/smarterag/data/RGB/RGB master files/RGB_data_AAPF_experiment_401.xlsx")

paths.hsi <-
c("/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_374.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_375.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_378.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_380.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_390.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_391.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_396.xlsx",
"/depot/smarterag/data/HSI/Master files/Hyperspectral_data_AAPF_experiment_401.xlsx")



# Unwanted data (incomplete scanning, demo for tour, etc.)
# See the interim results from this script (DataExplore.r)
#-------------------------------------------------------
# EXP ID TREATMENT DFP SCAN_DATETIME_BY15MIN NUM_SCANS
#-------------------------------------------------------
# 1
#374 20 -1 59 2023-10-30 11:15:00 4

# 2
#374 50-3 27 2023-09-28 09:15:00 2

# 3
#374 50-3 27 2023-09-28 09:00:00 24
# or
#374 50-3 32 2023-10-03 15:00:00 24

# 4
#374 50-4 27 2023-09-28 09:45:00 24
# or
#374 50-4 32 2023-10-03 15:00:00 24

# 5
#396 20-1 55 2024-04-29 12:00:00 1
#-------------------------------------------------------
df.unwanted <- data.frame(
EXP_ID=c(374, 374, 374, 374, 396),
TREATMENT=c("20 -1", "50-3", "50-3", "50-4", "20-1"),
SCAN_DATETIME_BY15MIN=c("2023-10-30 11:15:00",
"2023-09-28 09:15:00",
"2023-09-28 09:00:00",
"2023-09-28 09:45:00",
"2024-04-29 12:00:00"),
stringsAsFactors=FALSE)


# Treatment name change
df.convert.treatment <- data.frame(
from=c("20 -1", "20-2", "50-3", "50-4", "20-1", "20-2"),
to =c("20", "20", "80", "80", "20", "20"),
stringAsFactors=FALSE)




Expand All @@ -30,14 +97,13 @@ regex.col.hsi <- paste("EXP ID", "POT_BARCODE", "VARIETY", "TREATMENT",
sep="|")

# Column names to be disregarded
nouse.col.rgb <- c("Filename", "SCAN_TIME", "SCAN_DATE")
nouse.col.rgb <- c("Filename")
nouse.col.hsi <- c("Filename-VNIR-SIDE", "Filename-VNIR-TOP",
"Filename-SWIR-SIDE", "Filename-SWIR-TOP",
"SCAN_TIME", "SCAN_DATE")
"Filename-SWIR-SIDE", "Filename-SWIR-TOP")

# Column names used as identifier
id.vars <- c("EXP ID", "POT_BARCODE", "VARIETY",
"TREATMENT", "DFP", "View", "frame_nr")
# Column names used as interim identifier
id.vars <- c("EXP ID", "POT_BARCODE", "SCAN_TIME", "SCAN_DATE",
"VARIETY", "TREATMENT", "DFP", "View", "frame_nr")



Expand All @@ -47,9 +113,11 @@ id.vars <- c("EXP ID", "POT_BARCODE", "VARIETY",
mat <- matrix(ncol=0, nrow=0)
df.rgb <- data.frame(mat)

# Import rgb data
for (path.rgb in paths.rgb){
for (tab.rgb in excel_sheets(path = path.rgb)){
if (tab.rgb=="PPEW") next
if (tab.rgb=="Information") next

# Read a rgb worksheet
temp <- read_excel(path.rgb, sheet=tab.rgb)
Expand All @@ -71,7 +139,6 @@ for (path.rgb in paths.rgb){

# Assign values in the id columns that didn't exist
for (col.to.add in cols.to.add){
# TODO
if (col.to.add == "View" && tolower(tab.rgb) =="top"){
temp["View"] = "Top"
}
Expand All @@ -98,13 +165,17 @@ message('Succeeded importing RGB data...')



# Create empty rgb dataframe


# Create empty hsi dataframe
mat <- matrix(ncol=0, nrow=0)
df.hsi <- data.frame(mat)

# Import hsi data
for (path.hsi in paths.hsi){
for (tab.hsi in excel_sheets(path = path.hsi)){
if (tab.hsi=="PPEW") next
if (tab.hsi=="Information") next

# Read a hsi worksheet
temp <- read_excel(path.hsi, sheet=tab.hsi)
Expand Down Expand Up @@ -148,15 +219,88 @@ message('Succeeded importing HSI data...')



# Get unique rgb measurement
df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ]

# Show summary of rgb data
message('\nRGB Data Acquisiton Summary\n')
print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP))

# Round down scan date and time down to nearest 15 minute and display
format <- "%Y-%m-%d %H:%M:%S"
df.rgb.perscan["SCAN_DATETIME"] =
as.POSIXct(paste(as.Date(df.rgb.perscan$SCAN_DATE), df.rgb.perscan$SCAN_TIME),
format=format)
df.rgb.perscan["SCAN_DATETIME_BY15MIN"] =
lubridate::round_date(df.rgb.perscan$SCAN_DATETIME, "15 minutes")
message('\nRGB Data Acquisiton Summary by Date and Time\n')
print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP, SCAN_DATETIME_BY15MIN))
message('\nPlease use [`EXP ID`, TREATMENT, SCAN_DATETIME_BY15MIN]')
message('to exclude data from incomplete scan session...')
message('Check df.unwanted in DataExplore.r to check how to exclude records...\n')




# Drop unwanted rgb data
df.rgb["SCAN_DATETIME"] =
as.POSIXct(paste(as.Date(df.rgb$SCAN_DATE), df.rgb$SCAN_TIME), format=format)
df.rgb["SCAN_DATETIME_BY15MIN"] =
lubridate::round_date(df.rgb$SCAN_DATETIME, "15 minutes")
for(i in 1:nrow(df.unwanted)){
row <- df.unwanted[i,]
df.rgb <- df.rgb %>% filter(!(`EXP ID` == row$EXP_ID &
TREATMENT == row$TREATMENT &
SCAN_DATETIME_BY15MIN == row$SCAN_DATETIME_BY15MIN))
}
# Show data acquisiton summary of filtering out unwanted data
df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ]
message('\nRGB Data Acquisiton Summary (Unwanted Data Filtered)\n')
print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP))





# Convert treatment attributes
for(i in 1:nrow(df.convert.treatment)){
row <- df.convert.treatment[i,]
df.rgb$TREATMENT[df.rgb$TREATMENT==row$from] <- row$to
}
# Show data acquisiton summary of filtering out unwanted data
df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ]
message('\nRGB Data Acquisiton Summary (Treatment Info Updated)\n')
print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP))





# Merge RGB and HSI data
# DFP is always smaller than 365 (to prevent case with wrong year input)
df.rgb$DFP = df.rgb$DFP %% 365
# Show data acquisiton summary of filtering out unwanted data
df.rgb.perscan <- df.rgb[which(df.rgb$View=="Top" & df.rgb$variable=="Surface"), ]
message('\nRGB Data Acquisiton Summary (DFP is always lower than 365)\n')
print(count(df.rgb.perscan, `EXP ID`, TREATMENT, DFP))





# Set Growth stage
# frame_nr 0-11 (normal) , 0-23 for one scan
# For "RGB-SideAll", change column names Frame0-11 to major, major+30...
# Remove SCAN_TIME SCAN_DATE SCAN_DATETIME SCAN_DATETIME_BY15MIN

# Do the same for df.hsi
######### Combine rgb and hsi data
#########df <- rbind(df.rgb, df.hsi)






# Change DFPs to growth stage



Expand Down

0 comments on commit 007f882

Please sign in to comment.