---
title: "LA Dataset"
output:
  html_document:
    df_print: paged
  pdf_document: default
---

## Focus on 'Partitioning disks' and 'Maximal disks' stages.
```{r include=F}
library(tidyverse)
library(knitr)
library(kableExtra)
nExperiments = 180
ylim = 1150
```

```{r echo=F}
fields3 = c("Timestamp", "Tag", "appId", "Executors", "Cores", "Status", "Duration", "Stage", "Time", "Load", "Interval")
data_path = "~/Documents/PhD/Research/Scripts/R/R19/Plots/nohupMF2.txt"
stages = enframe(read_lines(data_path), name = "n", value = "line") %>% select(line) %>% 
  filter(grepl("\\|MF\\|", line)) %>%
  filter(grepl("END", line)) %>% separate(line, into = fields3, sep = "\\|") %>%
  filter(grepl("[A-F]\\.", Stage)) %>%
  mutate(Time = as.numeric(Time), Load = as.numeric(Load)) %>%
  select(appId, Stage, Time, Load) %>% mutate(id = rep(1:nExperiments, each = 6))

submit = enframe(read_lines(data_path), name = "n", value = "line") %>% select(line) %>% 
  filter(grepl("spark-submit", line)) %>% separate(line, into=paste0("A", 1:16), sep = "--") %>% 
  mutate(A11 = str_trim(A11), A12 = str_trim(A12)) %>%
  separate(A11, c(NA,"Instant"), sep = " ") %>% separate(A12, c(NA,"CellSize"), sep = " ") %>%
  select(Instant, CellSize) %>% mutate(Instant = as.numeric(Instant), CellSize = as.numeric(CellSize)) %>% 
  mutate(id = 1:nExperiments)
stage_names = tibble(StageId = c("A","B","C","D","E","F"), StageName = c("Partitioning points", "Finding pairs", "Computing centers", "Finding disks", "Partitioning disks", "Maximal disks"))
data = stages %>% inner_join(submit, by = "id") %>%
  group_by(appId, Instant, CellSize, Stage) %>% summarise(Time = mean(Time), Output = min(Load)) %>%
  ungroup() %>% separate(Stage, c("StageId", NA), sep = "\\.") %>% inner_join(stage_names, by = "StageId") %>%
  mutate(Stage = paste0(StageId,".",StageName)) %>% select(appId, Instant, CellSize, Stage, Time, Output)
head(data, n=nExperiments)
data$Input = c(0, data$Output[1:(length(data$Output) - 1)])
```

### Partitioning disks stage
```{r echo=F, fig.width=10}
  data0 = data %>% filter(Stage == "E.Partitioning disks") %>% select(Instant, CellSize, Input, Output, Time)
  
  p = ggplot(data = data0, aes(x = factor(Instant), y = Time, fill = factor(CellSize))) +
    geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + ylim(0, ylim) +
    labs(x="Time instant", y="Time [s]", title="Execution time of 'Maximal disk' stage by cell size", fill="Cell Size") 
  plot(p)
```

### Maximal disk stage
```{r echo=F, fig.width=10}
  data0 = data %>% filter(Stage == "F.Maximal disks") %>% select(Instant, CellSize, Input, Output, Time)
  
  p = ggplot(data = data0, aes(x = factor(Instant), y = Time, fill = factor(CellSize))) +
    geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + ylim(0, ylim) +
    labs(x="Time instant", y="Time [s]", title="Execution time of 'Maximal disk' stage by cell size", fill="Cell Size") 
  plot(p)
```

### Total time
```{r echo=F, fig.width=10}
data3 = data %>% group_by(appId, Instant, CellSize) %>% summarise(Time = sum(Time))
p = ggplot(data = data3, aes(x = factor(Instant), y = Time, fill = factor(CellSize))) +
  geom_bar(stat="identity", position=position_dodge(width = 0.75), width = 0.7) + ylim(0, ylim) +
  labs(x="Time instant", y="Time [s]", title="MF algorithm execution time by cell size at different time instants", fill="Cell Size") 
plot(p)
```