library(tidyverse)
library(janitor)
###################
STRUCTURES <- read_csv('https://dwr.state.co.us/Rest/GET/api/v2/structures/?format=csv&dateFormat=dateOnly&fields=wdid%2CciuCode%2CstructureType%2CwaterSource%2CassociatedCaseNumbers%2CassociatedPermits%2CassociatedContacts%2CwaterDistrict%2Csubdistrict%2Ccounty%2CmanagementDistrictName%2Clatdecdeg%2Clongdecdeg&division=3',skip=2) %>% clean_names() %>% mutate(wdid=as.character(wdid))
WELLS <- STRUCTURES %>% filter(structure_type=='WELL')
WELLS <- WELLS %>% filter(ciu_code %in% c('A','H')) #Keep only active wells or wells with historic records of pumping
WELLS <- WELLS %>% select( wdid,contacts=associated_contacts,latitude=latdecdeg,longitude=longdecdeg) 

#Read the data files which include the current list of wells by subdistrict, to create indicator variables. This is the list of wells currently in the subdistrict using hydrobase search filter. Note that the time of entry is not found here and added later.
SBD_LINK <- rbind(read_csv("Data/SBD_Data/StructureList_SBD1.csv") %>% clean_names %>% mutate(sbd=1) %>% select(wdid,sbd),
read_csv("Data/SBD_Data/StructureList_SBD2.csv")%>% clean_names %>% mutate(sbd=2) %>% select(wdid,sbd),
read_csv("Data/SBD_Data/StructureList_SBD3.csv")%>% clean_names %>% mutate(sbd=3) %>% select(wdid,sbd),
read_csv("Data/SBD_Data/StructureList_SBD4.csv")%>% clean_names %>% mutate(sbd=4) %>% select(wdid,sbd),
read_csv("Data/SBD_Data/StructureList_SBD5.csv")%>% clean_names %>% mutate(sbd=5) %>% select(wdid,sbd),
read_csv("Data/SBD_Data/StructureList_SBD6.csv")%>% clean_names %>% mutate(sbd=6) %>% select(wdid,sbd)) %>% mutate(wdid=as.character(wdid))
SBD_LINK <- SBD_LINK %>% pivot_wider(values_from=sbd,names_from=sbd,names_prefix="SBD") %>% replace(is.na(.), 0)
SBD_LINK[2:7][SBD_LINK[2:7]>0] <- 1
SBD1_WELLS <- 	SBD_LINK %>% filter(SBD1==1) %>% select(wdid)
############Data from each of the Annuall Replacement Plans or in the case of 2000 the Plan of Water Management. A list of wells in Subdistrict one was collected to create a time depdent indicator for being in SBD1.
SBD1_WELLS <- SBD1_WELLS %>% left_join( (read_csv("Data/ARP/SBD1_Wells.csv")%>% group_by(wdid) %>% filter(year==min(year)) %>% mutate(wdid=as.character(wdid)) %>% rename(SBD1_year=year) %>% ungroup))
WELLS <- WELLS[!(WELLS$wdid %in% c(SBD1_WELLS  %>% filter(is.na(SBD1_year)) %>% pull(wdid))),]#Drop odd records which were never in the SBD1 reports but are listed as SBD1 wells
SBD1_WELLS <-  SBD1_WELLS  %>% filter(!is.na(SBD1_year)) 


#######Original Wells based on map
	#This data comes from a manual map of SBD1 in 2009 made in QGIS. Wells are intersected with the hand drawn layer. The wells identified as being in the SBD1 region at this time should be assumed to have entered the subdistrict at the start of the program (2006 with first pumping data in 2009). This fixes the problem that some wells are identified as being in SBD1 in ARP reports but only because they were missed in earlier reports. It looks like most updated wells were missed due to category issues, such as not knowing the well was used for agriculture. This process should find most wells that were enrolled at the start, and later picked up so as to treat them as 2006/2009 wells instead of treating them as having entered in say 2016 or 2020 when they were first picked up by the SBD1 list. 
SBD1_FROM_MAP <- read_csv("Data/SBD_Data/SBD1_Map_Wells.csv") %>% mutate(wdid=as.character(wdid)) %>% mutate(IN_EXTENT=1) 
SBD1_WELLS <- SBD1_FROM_MAP  %>% full_join(SBD1_WELLS) %>% mutate(IN_EXTENT=ifelse(is.na(IN_EXTENT),0,1))
SBD1_WELLS <- SBD1_WELLS %>% filter(!is.na(SBD1_year))%>% mutate(SBD1_year=ifelse(IN_EXTENT==1,2011,SBD1_year)) %>% select(-IN_EXTENT)
SBD_LINK <- SBD1_WELLS  %>% full_join(SBD_LINK) %>% mutate(SBD1_year=ifelse(is.na(SBD1_year),Inf,SBD1_year),SBD1=ifelse(SBD1_year==Inf,0,1)) #Make sure both data sets conform. 
######################Static Data
STATIC_DATA <- 	WELLS%>% left_join(SBD_LINK) %>% mutate(SBD1_year=ifelse(is.na(SBD1_year),Inf,SBD1_year))
STATIC_DATA[,-1:-5] <- STATIC_DATA[,-1:-5] %>% replace(is.na(.), 0) #Include subdistrict indicators
DITCH_DATA <- readRDS("Data/Crop_Parcel_Data/Well_Ditch_Link.rds")
STATIC_DATA <- STATIC_DATA %>% left_join(DITCH_DATA )
DITCH_COL_NAMES <-colnames(STATIC_DATA)[grep("ditch",colnames(STATIC_DATA))]
STATIC_DATA[,DITCH_COL_NAMES] <- STATIC_DATA[,DITCH_COL_NAMES]%>% replace(is.na(.),0)

STATIC_DATA <- STATIC_DATA%>% left_join(read_csv("Data/Output_Data/Crops_Before_2009.csv") %>% mutate(wdid=as.character(wdid))) #Add crop data
STATIC_DATA$CROPS_PRE_2009 <- ifelse(is.na(STATIC_DATA$per_alfalfa),0,1) #Make an indicator to tell if crops were grown in 2002 or 2005, or if no crop data was available.
STATIC_DATA[-1:-5] <- STATIC_DATA[-1:-5] %>% replace(is.na(.), 0)
###Make sure all CREP wells are included in SBD1
PROGRAM_WELLS <- c(read_csv("Data/CREP_and_Fallow/Cleaned/Fallow_Program_Data/Bill_SB22_Fallow_Payments.csv")$wdid,
read_csv("Data/CREP_and_Fallow/Cleaned/Fallow_Program_Data/CREP.csv")$wdid,
read_csv("Data/CREP_and_Fallow/Cleaned/Fallow_Program_Data/SBD1_Half_Fallow_Program.csv")$wdid,
read_csv("Data/CREP_and_Fallow/Cleaned/Fallow_Program_Data/SBD1_Temporary_Fallow_Payments.csv")$wdid)
STATIC_DATA[STATIC_DATA$wdid %in% PROGRAM_WELLS,"SBD1_year"] <- 2011
STATIC_DATA[STATIC_DATA$wdid %in% PROGRAM_WELLS,"SBD1"] <- 1
AUG <- read_csv("Data/Augmentation/Augmentation.csv") %>% mutate(wdid=as.character(wdid)) %>% select(wdid)
STATIC_DATA$augmentation_well <- 0
STATIC_DATA[STATIC_DATA$wdid %in% AUG, "augmentation_well" ] <- 1


write_csv(STATIC_DATA,"Data/Output_Data/Well_Level_Static_Data.csv")
saveRDS(STATIC_DATA,"Data/Output_Data/Well_Level_Static_Data.rds")



###########Collect Pumping Data
LIST <- WELLS %>% pull(wdid) %>% unique 
ALL <- length(LIST )

if(file.exists("./Data/Output_Data/Div3_Pumping_Data.csv")){LIST <-LIST[-which(LIST %in%	t(read.csv("./Data/Output_Data/Div3_Pumping_Data.csv")[,1] %>% unique))]}
CURRENT <- length(LIST )

WITH_API_KEY <- FALSE 
for(C_WELL in LIST ){
	if(WITH_API_KEY){try(C_DAT <-	read_csv(paste0('https://dwr.state.co.us/Rest/GET/api/v2/structures/divrec/divrecyear/?format=csv&dateFormat=dateOnly&fields=wdid%2CwcIdentifier%2CdataMeasDate%2CdataValue&measUnits=ACFT&wcIdentifier=*U:1*&wdid=',C_WELL,'&apiKey=PwevUQJCStcYZfqrOYbuyztmNPlUJWby'),skip=2) %>% select(wdid,year=datameasuate,pumping=dataValue,wcIdentifier))} else{
try(C_DAT <-	read_csv(paste0('https://dwr.state.co.us/Rest/GET/api/v2/structures/divrec/divrecyear/?format=csv&dateFormat=dateOnly&fields=wdid%2CwcIdentifier%2CdataMeasDate%2CdataValue&measUnits=ACFT&wcIdentifier=*U:1*&wdid=',C_WELL),skip=2) %>% select(wdid,year=dataMeasDate,pumping=dataValue,wcIdentifier))
	}
if(exists("C_DAT")){	if(C_DAT[[1,1]]==C_WELL){write_csv(C_DAT,append=TRUE,file="./Data/Output_Data/Div3_Pumping_Data.csv")}}
	#closeAllConnections()
}
PUMPING <- read_csv("./Data/Output_Data/Div3_Pumping_Data.csv") %>% unique %>% replace(is.na(.), 0)

if(colnames(PUMPING)[1]!='wdid'){PUMPING <- read_csv("./Data/Output_Data/Div3_Pumping_Data.csv",col_names=c("wdid","year","AF")) %>% unique} #See if the names were already added, or if they need renamed
POST_LIST <- PUMPING %>% pull(wdid) %>% unique
PUMPING$ROW <- 1:nrow(PUMPING)
PUMPING <- PUMPING %>% pivot_wider(values_from=AF,names_from=year)%>% group_by(wdid) %>% summarize(across(as.character(2009:2025),\(x) mean(x, na.rm = TRUE))) %>% pivot_longer(-wdid,names_to='year',values_to='AF') %>% mutate(wdid=as.character(wdid)) %>% unique #Pivot to add zeros when a year is missing data
write_csv(PUMPING,file="./Data/Output_Data/Div3_Pumping_Data.csv")
ALL_DATA <- PUMPING %>% left_join(STATIC_DATA) %>% clean_names()
write_csv(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.csv")
print("Script 3: Process all data into panel completed")

