library(tidyverse)
library(janitor)

###########Collect Pumping Data
WELL_DATA <- read_csv("Data/Structure_Data/Structures_with_Diversions.csv")%>% clean_names() %>% mutate(wdid=as.character(wdid)) %>% select(wdid,contacts,latitude,longitude) #Start with well data
STATIC_DATA <- readRDS("Data/Output_Data/Well_Level_Static_Data.rds")
FALLOW_PROGRAM_DATA <- readRDS("Data/Output_Data/Fallow_Program_Data.rds")


LIST <- WELL_DATA %>% pull(wdid) %>% unique 
ALL <- length(LIST )

if(file.exists("./Data/Output_Data/Div3_Pumping_Data.csv")){LIST <-LIST[-which(LIST %in%	t(read.csv("./Data/Output_Data/Div3_Pumping_Data.csv")[,1] %>% unique))]}
CURRENT <- length(LIST )

WITH_API_KEY <- FALSE 
for(C_WELL in LIST ){
	if(WITH_API_KEY){try(C_DAT <-	read_csv(paste0('https://dwr.state.co.us/Rest/GET/api/v2/structures/divrec/divrecyear/?format=csv&dateFormat=dateOnly&fields=wdid%2CdataMeasDate%2CdataValue&measUnits=ACFT&wcIdentifier=*Total+(Diversion)*&wdid=',C_WELL,'&apiKey=PwevUQJCStcYZfqrOYbuyztmNPlUJWby'),skip=2) %>% select(wdid,year=datameasuate,pumping=dataValue))} else{
try(C_DAT <-	read_csv(paste0('https://dwr.state.co.us/Rest/GET/api/v2/structures/divrec/divrecyear/?format=csv&dateFormat=dateOnly&fields=wdid%2CdataMeasDate%2CdataValue&measUnits=ACFT&wcIdentifier=*Total+(Diversion)*&wdid=',C_WELL),skip=2) %>% select(wdid,year=dataMeasDate,pumping=dataValue))
	}
if(exists("C_DAT")){	if(C_DAT[[1,1]]==C_WELL){write_csv(C_DAT,append=TRUE,file="./Data/Output_Data/Div3_Pumping_Data.csv")}}
	#closeAllConnections()
}
PUMPING <- read_csv("./Data/Output_Data/Div3_Pumping_Data.csv") %>% unique %>% replace(is.na(.), 0)

if(colnames(PUMPING)[1]!='wdid'){PUMPING <- read_csv("./Data/Output_Data/Div3_Pumping_Data.csv",col_names=c("wdid","year","AF")) %>% unique} #See if the names were already added, or if they need renamed
POST_LIST <- PUMPING %>% pull(wdid) %>% unique
PUMPING$ROW <- 1:nrow(PUMPING)
PUMPING <- PUMPING %>% pivot_wider(values_from=AF,names_from=year)%>% group_by(wdid) %>% summarize(across(as.character(2009:2025),\(x) mean(x, na.rm = TRUE))) %>% pivot_longer(-wdid,names_to='year',values_to='AF') %>% mutate(wdid=as.character(wdid)) %>% unique #Pivot to add zeros when a year is missing data
#One value is slightly negative
PUMPING$AF <- ifelse(PUMPING$AF<0,0,PUMPING$AF)
write_csv(PUMPING,file="./Data/Output_Data/Div3_Pumping_Data.csv")

ALL_DATA <- PUMPING %>% left_join(STATIC_DATA) %>% clean_names()
DYNAMIC_DATA <- PUMPING %>% left_join(FALLOW_PROGRAM_DATA)%>% replace(is.na(.), 0)
ALL_DATA <- DYNAMIC_DATA %>% left_join(STATIC_DATA) 

write_csv(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.csv")
saveRDS(ALL_DATA,file="./Data/Output_Data/Full_Data_Set.rds")


