[DATALAD] Recorded changes

code/models/blocks/confusion_inverse_model_binomial_hurdle.stan

@@ -9,26 +9,29 @@ real inverse_model_lpdf(array [] matrix actual_confusion,
     //array [] vector actual_fp_rate,
     matrix mus,
     matrix etas,
-    real p//,
+    vector p//,
     //vector mus_fp,
     //vector alphas_fp
     ) {
         real ll = 0;
 
         vector [4] expect;
+        vector [4] sd;
 
         for (k in start:end) {
             expect = rep_vector(0, 4);
+            sd = rep_vector(0, 4);
 
             for (i in 1:n_classes) {
                 ll += beta_proportion_lpdf(actual_confusion[k-start+1,i] | mus[i,:], etas[i,:]);
                 //ll += gamma_lpdf(actual_fp_rate[k] | alphas_fp, alphas_fp./mus_fp);
                 
-                expect[i] = (2-p)*dot_product(truth_vocs[k,:], actual_confusion[k-start+1,:,i].*(1-actual_confusion[k-start+1,:,i]));
+                expect[i] = dot_product(truth_vocs[k,:], (2-p).*actual_confusion[k-start+1,:,i]);
+                sd[i] = dot_product(truth_vocs[k,:], (2-p).*(actual_confusion[k-start+1,:,i].*(1-actual_confusion[k-start+1,:,i])));
                 //expect[i] += actual_fp_rate[k,i] * duration;
             }
             
-            ll += normal_lpdf(vocs[k,:] | expect, sqrt(expect));
+            ll += normal_lpdf(vocs[k,:] | expect, sqrt(sd));
         }
 
         return ll;

code/models/blocks/confusion_model_binomial_hurdle_fast.stan

@@ -7,7 +7,7 @@ real confusion_model_lpdf(array[] matrix lambda,
     array[] int group,
     array[] real age,
     array[] real clip_duration,
-    real p
+    vector p
 ) {
     real ll = 0;
     vector [4] dp;
@@ -34,19 +34,19 @@ real confusion_model_lpdf(array[] matrix lambda,
                     // (as opposed to two vocs)
                     for (chi1 in max(0, 2*chi_d-algo[k,i]):chi_d) {
                         int chi = 2*chi_d-chi1;
-                        bp[1] = chi_d==0?0:binomial_lpmf(chi1 | chi_d, p);
+                        bp[1] = chi_d==0?0:binomial_lpmf(chi1 | chi_d, p[1]);
 
                         for (och_d in 0:(truth[k,2]>0?min(truth[k,2], algo[k,i]-chi):0)) {
                             dp[2] = truth[k,2]==0?0:binomial_lpmf(och_d | truth[k,2], lambda[lg,2,i]);
                             for (och1 in max(0, 2*och_d-(algo[k,i]-chi)):och_d) {
                                 int och = 2*och_d-och1;
-                                bp[2] = och_d==0?0:binomial_lpmf(och1 | och_d, p);
+                                bp[2] = och_d==0?0:binomial_lpmf(och1 | och_d, p[2]);
 
                                 for (fem_d in 0:(truth[k,3]>0?min(truth[k,3], algo[k,i]-chi-och):0)) {
                                     dp[3] = truth[k,3]==0?0:binomial_lpmf(fem_d | truth[k,3], lambda[lg,3,i]);
                                     for (fem1 in max(0, 2*fem_d-(algo[k,i]-chi-och)):fem_d) {
                                         int fem = 2*fem_d-fem1;
-                                        bp[3] = fem_d==0?0:binomial_lpmf(fem1 | fem_d, p);
+                                        bp[3] = fem_d==0?0:binomial_lpmf(fem1 | fem_d, p[3]);
 
                                         for (mal_d in 0:(truth[k,4]>0?min(truth[k,4], algo[k,i]-chi-och-fem):0)) {
                                             dp[4] = truth[k,4]==0?0:binomial_lpmf(mal_d | truth[k,4], lambda[lg,4,i]);
@@ -54,7 +54,7 @@ real confusion_model_lpdf(array[] matrix lambda,
                                                 int mal = 2*mal_d-mal1;
                                                 int delta = algo[k,i] - (mal+fem+och+chi);
                                                 if (delta==0) {
-                                                    bp[4] = mal_d==0?0:binomial_lpmf(mal1 | mal_d, p);
+                                                    bp[4] = mal_d==0?0:binomial_lpmf(mal1 | mal_d, p[4]);
                                                     log_contrib_comb[n] += sum(dp+bp);
                                                     n = n+1;
                                                 }

code/models/blocks/confusion_model_parameters_binomial_hurdle.stan

@@ -1,4 +1,4 @@
 matrix<lower=1>[n_classes,n_classes] etas;
 matrix<lower=0,upper=1>[n_classes,n_classes] mus;
 array [n_groups] matrix<lower=0,upper=1>[n_classes,n_classes] lambda;
-real<lower=0,upper=1> p;
+vector<lower=0,upper=1>[n_classes,n_classes] p;

code/models/dev_siblings_binomial_hurdle_fast.stan

@@ -0,0 +1,130 @@
+functions {
+    #include "blocks/confusion_model_binomial_hurdle_fast.stan"
+    #include "blocks/confusion_inverse_model_binomial_hurdle.stan"
+    #include "blocks/behavior_model_truth.stan"
+}
+
+// TODO
+// use speech rates to set priors on truth_vocs
+data {
+    int<lower=1> n_classes; // number of classes
+
+    // analysis data block
+    int<lower=1> n_recs;
+    int<lower=1> n_children;
+
+    array[n_recs] int<lower=1> children;
+    array[n_recs] real<lower=1> age;
+    array[n_recs] int<lower=-1> siblings;
+    array[n_recs, n_classes] int<lower=0> vocs;
+    array[n_children] int<lower=1> corpus;
+
+    real<lower=0> recs_duration;
+
+    // speaker confusion data block
+    int<lower=1> n_clips;   // number of clips
+    int<lower=1> n_groups; // number of groups
+    int<lower=1> n_corpora;
+    array [n_clips] int group;
+    array [n_clips] int conf_corpus;
+    array [n_clips,n_classes] int<lower=0> algo_total; // algo vocs attributed to specific speakers
+    array [n_clips,n_classes] int<lower=0> truth_total;
+    array [n_clips] real<lower=0> clip_duration;
+    array [n_clips] real<lower=0> clip_age;
+
+    int<lower=0> n_validation;
+
+    // actual speech rates
+    int<lower=1> n_rates;
+    int<lower=1> n_speech_rate_children;
+
+    array [n_rates,n_classes] int<lower=0> speech_rates;
+    array [n_rates] int group_corpus;
+    array [n_rates] real<lower=0> durations;
+    array [n_rates] real<lower=0> speech_rate_age;
+    array [n_rates] int<lower=-1> speech_rate_siblings;
+    array [n_rates] int<lower=1,upper=n_speech_rate_children> speech_rate_child;
+
+    // parallel processing
+    int<lower=1> threads;
+}
+
+transformed data {
+    vector<lower=0>[n_groups] recording_age;
+    array[n_speech_rate_children] int<lower=1> speech_rate_child_corpus;
+
+    array[n_children] int<lower=-1> child_siblings;
+    array[n_speech_rate_children] int<lower=-1> speech_rate_child_siblings;
+    int no_siblings = 0;
+    int has_siblings = 0;
+
+    for (c in 1:n_clips) {
+        recording_age[group[c]] = clip_age[c];
+    }
+
+    for (k in 1:n_rates) {
+        speech_rate_child_corpus[speech_rate_child[k]] = group_corpus[k];
+    }
+
+    for (k in 1:n_recs) {
+        child_siblings[children[k]] = siblings[k];
+    }
+
+    for (c in 1:n_children) {
+        if (child_siblings[c] == 0) {
+            no_siblings += 1;
+        }
+        else if (child_siblings[c] > 0) {
+            has_siblings += 1;
+        }
+    }
+
+    for (k in 1:n_rates) {
+        speech_rate_child_siblings[speech_rate_child[k]] = speech_rate_siblings[k];
+    }
+}
+
+parameters {
+    matrix<lower=0>[n_children,n_classes-1] mu_child_level;
+    vector [n_children] child_dev_age;
+    matrix<lower=0> [n_recs, n_classes] truth_vocs;
+    array [n_recs] matrix<lower=0,upper=1>[n_classes,n_classes] actual_confusion_baseline;
+
+    // confusion parameters
+    #include "blocks/confusion_model_parameters_binomial_hurdle.stan"
+
+    // behavior model parameters
+    #include "blocks/behavior_model_parameters.stan"
+
+    // parameters specific to human annotations
+    #include "blocks/human_annotations_parameters.stan"
+}
+
+model {
+    // inverse confusion model
+    target += reduce_sum(
+       inverse_model_lpdf, actual_confusion_baseline, 1,
+       n_recs, n_classes, recs_duration,
+       vocs, age,
+       truth_vocs, mus, etas, p//, mus_fp, etas_fp
+    );
+
+    // contribution of full recordings to the model of behavior
+    #include "blocks/behavior_observations_model.stan"
+
+    target += reduce_sum(
+        confusion_model_lpdf, lambda, 1,
+        n_classes, n_clips,
+        algo_total, truth_total, group, clip_duration, clip_age,
+        p//, lambda_fp
+    );
+
+    // priors on the nuisance parameters of the confusion model
+    #include "blocks/confusion_model_priors_binomial_hurdle.stan"
+
+    // priors on the hierarchical model of speech behavior
+    #include "blocks/behavior_model_priors.stan"
+
+    // human annotations contribution
+    #include "blocks/human_annotations.stan"
+}