diff --git a/.gitignore b/.gitignore index f318a13..a45ed26 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # ---> R -#Crop choice has very large raw files. These will be downloaded with a script so ignore them in git +#Crop choice has very large raw files. These will be downloaded with a script so ignore them in git. Same for structures Data/Crop_Choice/ +Data/Structure_Data/ #Ignore output data Data/Output_Data/ # diff --git a/1_Create_CREP_Data.r b/1_Create_CREP_Data.r index 9e0e0e8..0997292 100644 --- a/1_Create_CREP_Data.r +++ b/1_Create_CREP_Data.r @@ -135,4 +135,3 @@ ALL_PROGRAMS <- ALL_PROGRAMS %>% mutate(CREP_any=ifelse(CREP_perm+CREP_temp>0,1, dir.create("Data/Output_Data",showWarnings=FALSE,recursive=TRUE) write_csv(ALL_PROGRAMS,"Data/Output_Data/Fallow_Program_Data.csv") saveRDS(ALL_PROGRAMS,"Data/Output_Data/Fallow_Program_Data.rds") -print("Script 1: Create CREP data completed") diff --git a/3_Download_Crop_Choice_Data_from_Cloud.r b/3_Download_Crop_Choice_Data_from_Cloud.r deleted file mode 100644 index 191fadb..0000000 --- a/3_Download_Crop_Choice_Data_from_Cloud.r +++ /dev/null @@ -1,12 +0,0 @@ -library(RCurl) -#Download very large crop choice files created using the Hyrdobase map file in QGIS. This has every crop and technology combination for all parcels in 2002, or 2005. This is used as the pre-treatment control for crop choice correlated with factors such as soil quality. -#These files are excluded from git due to the very large space requirment. Instead Alex Gebben has hosted them on a Pcloud drive, and provided public access, allowing git to ignore them but have R download at project start. - -#Location of the files -CROP_2002_URL <- 'https://def3.pcloud.com/DLZHyJ74J7ZfHere67ZCPjOZXZqxJG5kZ2ZZM3FZZTvmJZzYZsLZ3TZH807r8lSpP0MjbM4PoY9z7sxqCDy/IRRIG_2002.csv' -CROP_2005_URL <- 'https://def3.pcloud.com/DLZwyJ74J7Z7zere67ZCPjOZXZExJG5kZ2ZZM3FZZuVFHZ3YZnYZmgZ9u59W63pgNRX6L94jxED10e4TrYk/IRRIG_2005.csv' - -DEST_DIR <- "./Data/Crop_Choice/" -dir.create(DEST_DIR,showWarnings=FALSE) -download.file(CROP_2002_URL,destfile=paste0(DEST_DIR,"IRRIG_2002.csv")) -download.file(CROP_2005_URL,destfile=paste0(DEST_DIR,"IRRIG_2005.csv")) diff --git a/3_Download_Large_Data_Sets_from_Cloud.r b/3_Download_Large_Data_Sets_from_Cloud.r new file mode 100644 index 0000000..7a7b67e --- /dev/null +++ b/3_Download_Large_Data_Sets_from_Cloud.r @@ -0,0 +1,20 @@ +library(RCurl) +#Download very large files created using the Hyrdobase map file in QGIS. For example, some files have every crop and technology combination for all parcels in 2002, or 2005. This is used as the pre-treatment control for crop choice correlated with factors such as soil quality. +#These files are excluded from git due to the very large space requirment. Instead Alex Gebben has hosted them on a Pcloud drive, and provided public access, allowing git to ignore them but have R download at project start. +###########Crop choices +#Location of the files +CROP_2002_URL <- 'https://def3.pcloud.com/DLZHyJ74J7ZfHere67ZCPjOZXZqxJG5kZ2ZZM3FZZTvmJZzYZsLZ3TZH807r8lSpP0MjbM4PoY9z7sxqCDy/IRRIG_2002.csv' +CROP_2005_URL <- 'https://def3.pcloud.com/DLZwyJ74J7Z7zere67ZCPjOZXZExJG5kZ2ZZM3FZZuVFHZ3YZnYZmgZ9u59W63pgNRX6L94jxED10e4TrYk/IRRIG_2005.csv' + +CROP_DEST_DIR <- "./Data/Crop_Choice/" +dir.create(CROP_DEST_DIR,showWarnings=FALSE) +download.file(CROP_2002_URL,destfile=paste0(CROP_DEST_DIR,"IRRIG_2002.csv")) +download.file(CROP_2005_URL,destfile=paste0(CROP_DEST_DIR,"IRRIG_2005.csv")) + +#Structures with Diversions +STRUCTURE_DEST_DIR <- "./Data/Structure_Data/" +dir.create(STRUCTURE_DEST_DIR,showWarnings=FALSE) + +STRUCTURE_URL <- 'https://def4.pcloud.com/DLZirjX4J7ZKuc7n67ZCPjOZXZgYFG5kZ2ZZM3FZZdskHZrFZEFZpRZrlmLBhBcTsyd3JedyKPSczAIOsNy/Structures_with_Diversions.csv' +download.file(STRUCTURE_URL,destfile=paste0(STRUCTURE_DEST_DIR,"Structures_with_Diversions.csv")) + diff --git a/3_Process_Ditches.r b/4_Process_Ditches.r similarity index 95% rename from 3_Process_Ditches.r rename to 4_Process_Ditches.r index 7a4e6ee..cbdac0d 100644 --- a/3_Process_Ditches.r +++ b/4_Process_Ditches.r @@ -18,5 +18,4 @@ DITCH_WELL[,-1] <- ifelse(is.na(DITCH_WELL[,-1]),0,1) DITCH_WELL <- DITCH_WELL %>% mutate(wdid=as.character(wdid)) saveRDS(DITCH_WELL,"Data/Crop_Parcel_Data/Well_Ditch_Link.rds") -print("Script 2: Process ditch link data completed") diff --git a/4_Download_and_Process_Pumping_Data.r b/5_Download_and_Process_Pumping_Data.r similarity index 85% rename from 4_Download_and_Process_Pumping_Data.r rename to 5_Download_and_Process_Pumping_Data.r index 39db45c..c82d93d 100644 --- a/4_Download_and_Process_Pumping_Data.r +++ b/5_Download_and_Process_Pumping_Data.r @@ -3,7 +3,6 @@ library(janitor) ###########Collect Pumping Data WELL_DATA <- read_csv("Data/Structure_Data/Structures_with_Diversions.csv")%>% clean_names() %>% mutate(wdid=as.character(wdid)) %>% select(wdid,contacts,latitude,longitude) #Start with well data -STATIC_DATA <- readRDS("Data/Output_Data/Well_Level_Static_Data.rds") FALLOW_PROGRAM_DATA <- readRDS("Data/Output_Data/Fallow_Program_Data.rds") @@ -31,11 +30,4 @@ PUMPING <- PUMPING %>% pivot_wider(values_from=AF,names_from=year)%>% group_by(w PUMPING$AF <- ifelse(PUMPING$AF<0,0,PUMPING$AF) write_csv(PUMPING,file="./Data/Output_Data/Div3_Pumping_Data.csv") -ALL_DATA <- PUMPING %>% left_join(STATIC_DATA) %>% clean_names() -DYNAMIC_DATA <- PUMPING %>% left_join(FALLOW_PROGRAM_DATA)%>% replace(is.na(.), 0) -ALL_DATA <- DYNAMIC_DATA %>% left_join(STATIC_DATA) - -write_csv(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.csv") -saveRDS(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.rds") - diff --git a/5_Process_All_Cleaned_Data.r b/6_Process_All_Cleaned_Data.r similarity index 100% rename from 5_Process_All_Cleaned_Data.r rename to 6_Process_All_Cleaned_Data.r