Cleanup and file corrections

.gitignore

@@ -1,6 +1,7 @@
 # ---> R
-#Crop choice has very large raw files. These will be downloaded with a script so ignore them in git
+#Crop choice has very large raw files. These will be downloaded with a script so ignore them in git. Same for structures
 Data/Crop_Choice/
+Data/Structure_Data/
 #Ignore output data
 Data/Output_Data/
 #

1_Create_CREP_Data.r

@@ -135,4 +135,3 @@ ALL_PROGRAMS <- ALL_PROGRAMS %>% mutate(CREP_any=ifelse(CREP_perm+CREP_temp>0,1,
 dir.create("Data/Output_Data",showWarnings=FALSE,recursive=TRUE)
 write_csv(ALL_PROGRAMS,"Data/Output_Data/Fallow_Program_Data.csv")
 saveRDS(ALL_PROGRAMS,"Data/Output_Data/Fallow_Program_Data.rds")
-print("Script 1: Create CREP data completed")

3_Download_Crop_Choice_Data_from_Cloud.r

@@ -1,12 +0,0 @@
-library(RCurl)
-#Download very large crop choice files created using the Hyrdobase map file in QGIS. This has every crop and technology combination for all parcels in 2002, or 2005. This is used as the pre-treatment control for crop choice correlated with factors such as soil quality.
-#These files are excluded from git due to the very large space requirment. Instead Alex Gebben has hosted them on a Pcloud drive, and provided public access, allowing git to ignore them but have R download at project start.
-
-#Location of the files
-CROP_2002_URL <- 'https://def3.pcloud.com/DLZHyJ74J7ZfHere67ZCPjOZXZqxJG5kZ2ZZM3FZZTvmJZzYZsLZ3TZH807r8lSpP0MjbM4PoY9z7sxqCDy/IRRIG_2002.csv'
-CROP_2005_URL <- 'https://def3.pcloud.com/DLZwyJ74J7Z7zere67ZCPjOZXZExJG5kZ2ZZM3FZZuVFHZ3YZnYZmgZ9u59W63pgNRX6L94jxED10e4TrYk/IRRIG_2005.csv'
-
-DEST_DIR <- "./Data/Crop_Choice/"
-dir.create(DEST_DIR,showWarnings=FALSE)
-download.file(CROP_2002_URL,destfile=paste0(DEST_DIR,"IRRIG_2002.csv"))
-download.file(CROP_2005_URL,destfile=paste0(DEST_DIR,"IRRIG_2005.csv"))

3_Download_Large_Data_Sets_from_Cloud.r

@@ -0,0 +1,20 @@
+library(RCurl)
+#Download very large files created using the Hyrdobase map file in QGIS. For example, some files have every crop and technology combination for all parcels in 2002, or 2005. This is used as the pre-treatment control for crop choice correlated with factors such as soil quality.
+#These files are excluded from git due to the very large space requirment. Instead Alex Gebben has hosted them on a Pcloud drive, and provided public access, allowing git to ignore them but have R download at project start.
+###########Crop choices
+#Location of the files
+CROP_2002_URL <- 'https://def3.pcloud.com/DLZHyJ74J7ZfHere67ZCPjOZXZqxJG5kZ2ZZM3FZZTvmJZzYZsLZ3TZH807r8lSpP0MjbM4PoY9z7sxqCDy/IRRIG_2002.csv'
+CROP_2005_URL <- 'https://def3.pcloud.com/DLZwyJ74J7Z7zere67ZCPjOZXZExJG5kZ2ZZM3FZZuVFHZ3YZnYZmgZ9u59W63pgNRX6L94jxED10e4TrYk/IRRIG_2005.csv'
+
+CROP_DEST_DIR <- "./Data/Crop_Choice/"
+dir.create(CROP_DEST_DIR,showWarnings=FALSE)
+download.file(CROP_2002_URL,destfile=paste0(CROP_DEST_DIR,"IRRIG_2002.csv"))
+download.file(CROP_2005_URL,destfile=paste0(CROP_DEST_DIR,"IRRIG_2005.csv"))
+
+#Structures with Diversions
+STRUCTURE_DEST_DIR <- "./Data/Structure_Data/"
+dir.create(STRUCTURE_DEST_DIR,showWarnings=FALSE)
+
+STRUCTURE_URL <- 'https://def4.pcloud.com/DLZirjX4J7ZKuc7n67ZCPjOZXZgYFG5kZ2ZZM3FZZdskHZrFZEFZpRZrlmLBhBcTsyd3JedyKPSczAIOsNy/Structures_with_Diversions.csv'
+download.file(STRUCTURE_URL,destfile=paste0(STRUCTURE_DEST_DIR,"Structures_with_Diversions.csv"))
+

4_Process_Ditches.r

@@ -18,5 +18,4 @@ DITCH_WELL[,-1] <- ifelse(is.na(DITCH_WELL[,-1]),0,1)
 DITCH_WELL <- DITCH_WELL %>% mutate(wdid=as.character(wdid))
 
 saveRDS(DITCH_WELL,"Data/Crop_Parcel_Data/Well_Ditch_Link.rds")
-print("Script 2: Process ditch link data completed")
 

5_Download_and_Process_Pumping_Data.r

@@ -3,7 +3,6 @@ library(janitor)
 
 ###########Collect Pumping Data
 WELL_DATA <- read_csv("Data/Structure_Data/Structures_with_Diversions.csv")%>% clean_names() %>% mutate(wdid=as.character(wdid)) %>% select(wdid,contacts,latitude,longitude) #Start with well data
-STATIC_DATA <- readRDS("Data/Output_Data/Well_Level_Static_Data.rds")
 FALLOW_PROGRAM_DATA <- readRDS("Data/Output_Data/Fallow_Program_Data.rds")
 
 
@@ -31,11 +30,4 @@ PUMPING <- PUMPING %>% pivot_wider(values_from=AF,names_from=year)%>% group_by(w
 PUMPING$AF <- ifelse(PUMPING$AF<0,0,PUMPING$AF)
 write_csv(PUMPING,file="./Data/Output_Data/Div3_Pumping_Data.csv")
 
-ALL_DATA <- PUMPING %>% left_join(STATIC_DATA) %>% clean_names()
-DYNAMIC_DATA <- PUMPING %>% left_join(FALLOW_PROGRAM_DATA)%>% replace(is.na(.), 0)
-ALL_DATA <- DYNAMIC_DATA %>% left_join(STATIC_DATA) 
-
-write_csv(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.csv")
-saveRDS(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.rds")
-