Wgsmart 2022

README.md

@@ -1,9 +1,18 @@
 # 2018_smartdotsReport_template
 
-This template is used to create reports from smartdots events.  The settings for the report are contained in the [config.json](config.json) file.  Currently, the code requires direct access to the smartdots database and so must be run by someone with the appropriate access.  To run the template:
+This template is used to create reports from smartdots events.  The 
+settings for the report are contained in the [config.json](config.json) 
+file, in the bootstrap/initial/data folder.  Currently, the code requires 
+direct access to the smartdots database and so must be run by someone 
+with the appropriate access.  To run the template:
 
 ```r
 library(icesTAF)
+# download data
+taf.bootstrap(taf = TRUE)
+# run analysis and create reports
 sourceAll()
 ```
 
+To change the configuration and run the project, the script `worksheet.R`
+is provided to set up the config file when changeing eventID for example.

bootstrap/initial/data/config.json

@@ -1,11 +1,12 @@
 {
-  "event_id": 225,
+  "event_id": 347,
   "ma_method": "Mode",
-  "strata": "strata",
   "onlyApproved": true,
-  "summary_name": "SmartDots_Summary",
-  "summary_title": "SmartDots Summary",
-  "report_name": "SmartDots_Report",
-  "report_title": "SmartDots Report",
-  "report_tokens": "token_goes_here"
+  "summary_name": "SmartDots_Summary_Event_347",
+  "summary_title": "SmartDots Summary for event 347",
+  "report_name": "SmartDots_Report_Event_347",
+  "report_title": "SmartDots Report for event 347",
+  "report_tokens": "tokens goes here",
+  "mode_definition": "multistage",
+  "strata": ["strata"]
 }

bootstrap/smartdots_db.R

@@ -8,36 +8,31 @@
 library(icesTAF)
 library(jsonlite)
 
-
 # load configuration
-config <- read_json("../config.json", simplifyVector = TRUE)
+config <- read_json(taf.data.path("config.json"), simplifyVector = TRUE)
 
 # get data from api --------
 zipfile <- "smartdots_data.zip"
+download.file(
+  paste0("https://smartdots.ices.dk/download/DownloadEvent.ashx?Token=3F482827-282B-46E0-8C58-FBCF6C4B2C55&tblEventID=", config$event_id),
+  zipfile,
+  mode = "wb"
+)
 
-url <-
-  paste0(
-    "https://smartdots.ices.dk/download/DownloadEvent.ashx?",
-    "Token=", config$token,
-    "&tblEventID=", config$event_id
-  )
-
-download.file(url,
-              zipfile,  mode = "wb")
 files <- unzip(zipfile, list = TRUE)$Name
 files <- files[grep("*.csv", files)]
 unzip(zipfile, files = files, exdir = ".")
 
-# read in and write out again
 dist <- read.csv(files[grep("DotsDistances",  files)], stringsAsFactors = FALSE)
-ad <- read.csv(files[grep("Annotations",  files)], stringsAsFactors = FALSE)
+data <- read.csv(files[grep("Annotations",  files)], stringsAsFactors = FALSE)
 
 # delete downloaded data
 unlink(files)
+unlink(zipfile)
 
 # drop comments feild
-ad <- ad[,names(ad) != "Comment"]
+data <- data[, names(data) != "Comment"]
 
 # write out 'bootstrap' data tables
-write.taf(dist, "dist.csv", quote = TRUE)
-write.taf(ad, "data.csv", quote = TRUE)
+write.taf(dist, quote = TRUE)
+write.taf(data, quote = TRUE)

data.R

@@ -9,4 +9,3 @@ sourceTAF("data_checker.R")
 sourceTAF("data_processing.R")
 
 # done
-

data_checker.R

@@ -76,8 +76,10 @@ check_ad <- function(ad, what = "ad") {
              "** There are no advanced readers!                           **\n",
              "** the report scripts require there to be advanced readers. **"
       )
+
   }
 
+
   msg(check_text, "\n")
 }
 

data_processing.R

@@ -7,12 +7,11 @@
 
 library(icesTAF)
 library(jsonlite)
+library(plyr)
 library(dplyr)
 library(tidyr)
-require(lubridate)
-
-# create data directory
-mkdir("data")
+library(lubridate)
+library(stringr)
 
 # get utility functions
 source("utilities.R")
@@ -42,6 +41,7 @@ ad <-
     month = lubridate::month(parse_date_time(catch_date, '%d/%m/%Y %H:%M:%S'))
   })
 
+
 # if variables are missing add "missing"
 ad$ices_area[is.na(ad$ices_area) | ad$ices_area == ""] <- "missing"
 ad$stock[is.na(ad$stock) | ad$stock == ""] <- "missing"
@@ -50,6 +50,24 @@ ad$prep_method[is.na(ad$prep_method) | ad$prep_method == ""] <- "missing"
 # if variables are missing add "missing"
 dist$ices_area[is.na(dist$ices_area) | dist$ices_area == ""] <- "missing"
 
+# check or create the stratification here
+if (is.null(config$strata) || is.na(config$strata)) {
+  ad$strata = NA
+} else {
+  ad$strata = apply(as.data.frame(ad[, config$strata]), 1, function(x) {
+    paste(str_to_title(config$strata), x, sep = "_", collapse = "_and_")
+  })
+}
+
+
+
+estratos <- plyr::ddply(ad, .(SampleID), summarise, strata = unique(strata))
+dim(dist)
+dist <- dist[,!colnames(dist) %in% c("strata")]
+dist <- merge(dist, estratos, by.x="SampleID", by.y="SampleID")
+dim(dist)
+
+
 # if no advanced readers! make them all advanced
 if (all(ad$expertise == 0)) {
   msg("NOTE: all readers were Basic - all have been converted to Advanced")
@@ -59,18 +77,28 @@ if (all(ad$expertise == 0)) {
 # convert reader expertise
 ad$expertise <- c("Basic", "Advanced")[ad$expertise + 1]
 
-# if strata is missing, add "all"
-# has strata been set?
-if (all(is.na(ad$strata))) {
-  msg("NOTE: strata column not assigned - setting strata to 'all'")
-  ad$strata <- "all"
-  dist$strata <- "all"
-}
+# Assign weight to the readers based in their ranking-experience
+weight <- length(sort(unique(ad$reader_number))):1
+reader_number <- sort(unique(ad$reader_number))
+reader <- data.frame(reader_number = reader_number, weight_I = weight, weight_II = 1 / (1 + log(sort(weight, decreasing = FALSE) + 0.0000000001)))
+ad <- merge(ad, reader, by.x = "reader_number", by.y = "reader_number", all.x = TRUE)
+
 
 # Calculate modal ages and cv of modal age
-ad_long <- add_modalage(ad, config$ma_method)
-ad_long_ex <- add_modalage(ad[ad$expertise == "Advanced", ], config$ma_method)
+#message("perhaps change modal calculation functions to better name columns 0, 1, 2 ...")
+ad_long <- ad %>%
+  add_modal_trad(config$ma_method) %>%
+  add_modal_linearweight(config$ma_method) %>%
+  add_modal_negexpweight(config$ma_method)
+
+ad_long_adv <- ad[ad$expertise == "Advanced", ] %>%
+  add_modal_trad(config$ma_method) %>%
+  add_modal_linearweight(config$ma_method) %>%
+  add_modal_negexpweight(config$ma_method)
 
+# Choose the final mode (traditional, readers linear weight or negative exponential linear weight) based in the existence of multimodality or not.
+ad_long <- select_mode(ad_long, config$ma_method, config$mode_definition)
+ad_long_adv <- select_mode(ad_long_adv, config$ma_method, config$mode_definition)
 
 # prepare data in wbgr output format
 # IMAGE,1,2,3,4,5,6,7,8,9,10,11,12,13
@@ -79,7 +107,7 @@ ad_long_ex <- add_modalage(ad[ad$expertise == "Advanced", ], config$ma_method)
 # 6698256.jpg,1,1,1,1,1,-,2,1,-,2,-,1,-
 # 6698257.jpg,3,3,3,3,2,1,3,3,-,3,-,3,-
 readers <- sort(unique(ad$reader))
-webgr <- spread(ad_long[c("sample", "reader", "age")], key = reader, value = age)
+webgr <- spread(ad[c("sample", "reader", "age")], key = reader, value = age)
 webgr[] <- paste(unlist(webgr))
 webgr[webgr == "NA"] <- "-"
 webgr <-
@@ -89,10 +117,9 @@ webgr <-
 names(webgr) <- c("IMAGE", 1:length(readers))
 head(webgr)
 
-
 # write out input data tables for use later
-write.taf(dist, "data/dist.csv", quote = TRUE)
-write.taf(ad, "data/data.csv", quote = TRUE)
-write.taf(ad_long, "data/ad_long.csv", quote = TRUE)
-write.taf(ad_long_ex, "data/ad_long_ex.csv", quote = TRUE)
-write.taf(webgr, "data/WebGR_ages_all.csv", quote = TRUE)
+write.taf(dist, dir = "data", quote = TRUE)
+write.taf(ad, file = "data.csv", dir = "data", quote = TRUE)
+write.taf(ad_long, dir = "data", quote = TRUE)
+write.taf(ad_long_adv, dir = "data", quote = TRUE)
+write.taf(webgr, file = "WebGR_ages_all.csv", dir = "data", quote = TRUE)