#install.packages("devtools")
#devtools::install_github("anguswg-ucsb/cdssr")
#library(cdssr)
library(tidyverse)
library(janitor)
#install.packages("janitor")
#API <- 'PwevUQJCStcYZfqrOYbuyztmNPlUJWby'
	read_csv("Data/SBD_Data/StructureList_SBD2.csv")
SBD_LINK <- rbind(read_csv("Data/SBD_Data/StructureList_SBD1.csv") %>% mutate(SBD=1),
read_csv("Data/SBD_Data/StructureList_SBD2.csv") %>% mutate(SBD=2),
read_csv("Data/SBD_Data/StructureList_SBD3.csv") %>% mutate(SBD=3),
read_csv("Data/SBD_Data/StructureList_SBD4.csv") %>% mutate(SBD=4),
read_csv("Data/SBD_Data/StructureList_SBD5.csv") %>% mutate(SBD=5),
read_csv("Data/SBD_Data/StructureList_SBD6.csv") %>% mutate(SBD=6)) %>% clean_names %>% mutate(wdid=as.character(wdid))
SBD_LINK <- SBD_LINK %>% pivot_wider(values_from=sbd,names_from=sbd,names_prefix="SBD") %>% replace(is.na(.), 0)
SBD_LINK[2:7][SBD_LINK[2:7]>0] <- 1

#################
#STRUCTURES <- 'https://dwr.state.co.us/Rest/GET/api/v2/structures/?format=csv&fields=wdid%2CciuCode%2CstructureType&division=3&pageSize=500000&apiKey=PwevUQJCStcYZfqrOYbuyztmNPlUJWby'
#STRUCTURES<- read_csv(STRUCTURES,skip=2 )
#WELLS <- STRUCTURES %>% filter(structuretype=='WELL')

WELL_DATA <- read_csv("Data/Structure_Data/Structures_with_Diversions.csv")%>% clean_names()  %>% mutate(wdid=as.character(wdid)) %>% select(wdid,contacts) #Start with well data
STATIC_DATA <- WELL_DATA %>% left_join(SBD_LINK) %>% replace(is.na(.), 0) #Include subdistrict indicators
STATIC_DATA  <- STATIC_DATA %>% left_join(read_csv("Data/Output_Data/Ditch_Indicators.csv")%>% mutate(wdid=as.character(wdid)))%>% replace(is.na(.), 0)

STATIC_DATA <- STATIC_DATA%>% left_join(read_csv("Data/Output_Data/Crops_Before_2009.csv") %>% mutate(wdid=as.character(wdid))) #Add crop data
STATIC_DATA$CROPS_PRE_2009 <- ifelse(is.na(STATIC_DATA$per_alfalfa),0,1) #Make an indicator to tell if crops were grown in 2002 or 2005, or if not crop data was available.
STATIC_DATA <- STATIC_DATA %>% replace(is.na(.), 0)

write_csv(STATIC_DATA,"Data/Output_Data/Well_Level_Static_Data.csv")


###########Collect Pumping Data
LIST <- WELL_DATA %>% pull(wdid) %>% unique 
ALL <- length(LIST )

if(file.exists("./Data/Output_Data/Div3_Pumping_Data.csv")){LIST <-LIST[-which(LIST %in%	t(read.csv("./Data/Output_Data/Div3_Pumping_Data.csv")[,1] %>% unique))]}
CURRENT <- length(LIST )
ALL
CURRENT 

WITH_API_KEY <- FALSE 
for(C_WELL in LIST ){
	if(WITH_API_KEY){try(C_DAT <-	read_csv(paste0('https://dwr.state.co.us/Rest/GET/api/v2/structures/divrec/divrecyear/?format=csv&dateFormat=dateOnly&fields=wdid%2CdataMeasDate%2CdataValue&measUnits=ACFT&wcIdentifier=*Total+(Diversion)*&wdid=',C_WELL,'&apiKey=PwevUQJCStcYZfqrOYbuyztmNPlUJWby'),skip=2) %>% select(wdid,year=datameasuate,pumping=dataValue))} else{
try(C_DAT <-	read_csv(paste0('https://dwr.state.co.us/Rest/GET/api/v2/structures/divrec/divrecyear/?format=csv&dateFormat=dateOnly&fields=wdid%2CdataMeasDate%2CdataValue&measUnits=ACFT&wcIdentifier=*Total+(Diversion)*&wdid=',C_WELL),skip=2) %>% select(wdid,year=dataMeasDate,pumping=dataValue))
	}
if(exists("C_DAT")){	if(C_DAT[[1,1]]==C_WELL){write_csv(C_DAT,append=TRUE,file="./Data/Output_Data/Div3_Pumping_Data.csv")}}
	#closeAllConnections()
}
PUMPING <- read_csv("./Data/Output_Data/Div3_Pumping_Data.csv") %>% unique %>% replace(is.na(.), 0)

if(colnames(PUMPING)[1]!='wdid'){PUMPING <- read_csv("./Data/Output_Data/Div3_Pumping_Data.csv",col_names=c("wdid","year","AF")) %>% unique} #See if the names were already added, or if they need renamed
POST_LIST <- PUMPING %>% pull(wdid) %>% unique
PUMPING %>% group_by(wdid,year) %>% filter(n()>1) %>% arrange(wdid,year)
POST_LIST[1]
PUMPING$ROW <- 1:nrow(PUMPING)
PUMPING <- PUMPING %>% pivot_wider(values_from=AF,names_from=year)%>% group_by(wdid) %>% summarize(across(as.character(2009:2025),\(x) mean(x, na.rm = TRUE))) %>% pivot_longer(-wdid,names_to='year',values_to='AF') %>% mutate(wdid=as.character(wdid)) %>% unique #Pivot to add zeros when a year is missing data
write_csv(PUMPING,file="./Data/Output_Data/Div3_Pumping_Data.csv")
ALL_DATA <- PUMPING %>% left_join(STATIC_DATA) %>% clean_names()
write_csv(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.csv")