Calculates fit statistics from user data and writes it to a JSON file for importing into the common model repository.
calculateFitStat(
targetName,
targetPredicted,
validadedf = NULL,
traindf = NULL,
testdf = NULL,
type = "binary",
targetEventValue = 1,
path = NULL,
label.ordering = c(0, 1),
cutoff = 0.5,
noFile = FALSE
)target variable column name (actuals)
target variable column name. When type = "binary" it should be a probability.
data.frame where the first column in the yActual (labels/value) and the second is yPrediction (target probability)
data.frame where the first column in the yActual (labels/value) and the second is yPrediction (target probability)
data.frame where the first column in the yActual (labels/value) and the second is yPrediction (target probability)
"binary" or "interval"
if type = "binary" target class name for fit stat reference, if model is nominal, all other class will be counted as "not target"
directory where the JSON file is written; required when
noFile = FALSE
The default ordering (cf.details) of the classes can be changed by supplying a vector containing the negative and the positive class label. See ROCR::prediction()
cutoff to be used for calculation of miss classification for binary
if you don't want to write to a file, only the output
list that reflects the 'dmcas_fitstat.json'
'dmcas_fitstat.json' file written to path
df <- data.frame(label = sample(c(1,0), 6000, replace = TRUE),
prob = runif(6000),
partition = rep_len(1:3, 6000))
calculateFitStat(targetName = "label",
targetPredicted = "prob",
df[df$partition == 1, ],
df[df$partition == 2, ],
df[df$partition == 3, ],
noFile = TRUE)
#> $creationTimeStamp
#> NULL
#>
#> $modifiedTimeStamp
#> NULL
#>
#> $createdBy
#> NULL
#>
#> $modifiedBy
#> NULL
#>
#> $id
#> NULL
#>
#> $name
#> [1] "dmcas_fitstat"
#>
#> $description
#> NULL
#>
#> $revision
#> [1] 0
#>
#> $groupName
#> NULL
#>
#> $order
#> [1] 0
#>
#> $type
#> NULL
#>
#> $parameterMap
#> $parameterMap$`_RASE_`
#> $parameterMap$`_RASE_`$parameter
#> [1] "_RASE_"
#>
#> $parameterMap$`_RASE_`$type
#> [1] "num"
#>
#> $parameterMap$`_RASE_`$label
#> [1] "Root Average Squared Error"
#>
#> $parameterMap$`_RASE_`$length
#> [1] 8
#>
#> $parameterMap$`_RASE_`$order
#> [1] 7
#>
#> $parameterMap$`_RASE_`$values
#> [1] "_RASE_"
#>
#> $parameterMap$`_RASE_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_NObs_`
#> $parameterMap$`_NObs_`$parameter
#> [1] "_NObs_"
#>
#> $parameterMap$`_NObs_`$type
#> [1] "num"
#>
#> $parameterMap$`_NObs_`$label
#> [1] "Sum of Frequencies"
#>
#> $parameterMap$`_NObs_`$length
#> [1] 8
#>
#> $parameterMap$`_NObs_`$order
#> [1] 4
#>
#> $parameterMap$`_NObs_`$values
#> [1] "_NObs_"
#>
#> $parameterMap$`_NObs_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_GINI_`
#> $parameterMap$`_GINI_`$parameter
#> [1] "_GINI_"
#>
#> $parameterMap$`_GINI_`$type
#> [1] "num"
#>
#> $parameterMap$`_GINI_`$label
#> [1] "Gini Coefficient"
#>
#> $parameterMap$`_GINI_`$length
#> [1] 8
#>
#> $parameterMap$`_GINI_`$order
#> [1] 12
#>
#> $parameterMap$`_GINI_`$values
#> [1] "_GINI_"
#>
#> $parameterMap$`_GINI_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_GAMMA_`
#> $parameterMap$`_GAMMA_`$parameter
#> [1] "_GAMMA_"
#>
#> $parameterMap$`_GAMMA_`$type
#> [1] "num"
#>
#> $parameterMap$`_GAMMA_`$label
#> [1] "Gamma"
#>
#> $parameterMap$`_GAMMA_`$length
#> [1] 8
#>
#> $parameterMap$`_GAMMA_`$order
#> [1] 13
#>
#> $parameterMap$`_GAMMA_`$values
#> [1] "_GAMMA_"
#>
#> $parameterMap$`_GAMMA_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_formattedPartition_`
#> $parameterMap$`_formattedPartition_`$parameter
#> [1] "_formattedPartition_"
#>
#> $parameterMap$`_formattedPartition_`$type
#> [1] "char"
#>
#> $parameterMap$`_formattedPartition_`$label
#> [1] "Formatted Partition"
#>
#> $parameterMap$`_formattedPartition_`$length
#> [1] 12
#>
#> $parameterMap$`_formattedPartition_`$order
#> [1] 3
#>
#> $parameterMap$`_formattedPartition_`$values
#> [1] "_formattedPartition_"
#>
#> $parameterMap$`_formattedPartition_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_DataRole_`
#> $parameterMap$`_DataRole_`$parameter
#> [1] "_DataRole_"
#>
#> $parameterMap$`_DataRole_`$type
#> [1] "char"
#>
#> $parameterMap$`_DataRole_`$label
#> [1] "Data Role"
#>
#> $parameterMap$`_DataRole_`$length
#> [1] 10
#>
#> $parameterMap$`_DataRole_`$order
#> [1] 1
#>
#> $parameterMap$`_DataRole_`$values
#> [1] "_DataRole_"
#>
#> $parameterMap$`_DataRole_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_MCE_`
#> $parameterMap$`_MCE_`$parameter
#> [1] "_MCE_"
#>
#> $parameterMap$`_MCE_`$type
#> [1] "num"
#>
#> $parameterMap$`_MCE_`$label
#> [1] "Misclassification Rate"
#>
#> $parameterMap$`_MCE_`$length
#> [1] 8
#>
#> $parameterMap$`_MCE_`$order
#> [1] 8
#>
#> $parameterMap$`_MCE_`$values
#> [1] "_MCE_"
#>
#> $parameterMap$`_MCE_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_ASE_`
#> $parameterMap$`_ASE_`$parameter
#> [1] "_ASE_"
#>
#> $parameterMap$`_ASE_`$type
#> [1] "num"
#>
#> $parameterMap$`_ASE_`$label
#> [1] "Average Squared Error"
#>
#> $parameterMap$`_ASE_`$length
#> [1] 8
#>
#> $parameterMap$`_ASE_`$order
#> [1] 5
#>
#> $parameterMap$`_ASE_`$values
#> [1] "_ASE_"
#>
#> $parameterMap$`_ASE_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_MCLL_`
#> $parameterMap$`_MCLL_`$parameter
#> [1] "_MCLL_"
#>
#> $parameterMap$`_MCLL_`$type
#> [1] "num"
#>
#> $parameterMap$`_MCLL_`$label
#> [1] "Multi-Class Log Loss"
#>
#> $parameterMap$`_MCLL_`$length
#> [1] 8
#>
#> $parameterMap$`_MCLL_`$order
#> [1] 9
#>
#> $parameterMap$`_MCLL_`$values
#> [1] "_MCLL_"
#>
#> $parameterMap$`_MCLL_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_KS_`
#> $parameterMap$`_KS_`$parameter
#> [1] "_KS_"
#>
#> $parameterMap$`_KS_`$type
#> [1] "num"
#>
#> $parameterMap$`_KS_`$label
#> [1] "KS (Youden)"
#>
#> $parameterMap$`_KS_`$length
#> [1] 8
#>
#> $parameterMap$`_KS_`$order
#> [1] 10
#>
#> $parameterMap$`_KS_`$values
#> [1] "_KS_"
#>
#> $parameterMap$`_KS_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_KSPostCutoff_`
#> $parameterMap$`_KSPostCutoff_`$parameter
#> [1] "_KSPostCutoff_"
#>
#> $parameterMap$`_KSPostCutoff_`$type
#> [1] "num"
#>
#> $parameterMap$`_KSPostCutoff_`$label
#> [1] "ROC Separation"
#>
#> $parameterMap$`_KSPostCutoff_`$length
#> [1] 8
#>
#> $parameterMap$`_KSPostCutoff_`$order
#> [1] 16
#>
#> $parameterMap$`_KSPostCutoff_`$values
#> [1] "_KSPostCutoff_"
#>
#> $parameterMap$`_KSPostCutoff_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_DIV_`
#> $parameterMap$`_DIV_`$parameter
#> [1] "_DIV_"
#>
#> $parameterMap$`_DIV_`$type
#> [1] "num"
#>
#> $parameterMap$`_DIV_`$label
#> [1] "Divisor for ASE"
#>
#> $parameterMap$`_DIV_`$length
#> [1] 8
#>
#> $parameterMap$`_DIV_`$order
#> [1] 6
#>
#> $parameterMap$`_DIV_`$values
#> [1] "_DIV_"
#>
#> $parameterMap$`_DIV_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_TAU_`
#> $parameterMap$`_TAU_`$parameter
#> [1] "_TAU_"
#>
#> $parameterMap$`_TAU_`$type
#> [1] "num"
#>
#> $parameterMap$`_TAU_`$label
#> [1] "Tau"
#>
#> $parameterMap$`_TAU_`$length
#> [1] 8
#>
#> $parameterMap$`_TAU_`$order
#> [1] 14
#>
#> $parameterMap$`_TAU_`$values
#> [1] "_TAU_"
#>
#> $parameterMap$`_TAU_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_KSCut_`
#> $parameterMap$`_KSCut_`$parameter
#> [1] "_KSCut_"
#>
#> $parameterMap$`_KSCut_`$type
#> [1] "num"
#>
#> $parameterMap$`_KSCut_`$label
#> [1] "KS Cutoff"
#>
#> $parameterMap$`_KSCut_`$length
#> [1] 8
#>
#> $parameterMap$`_KSCut_`$order
#> [1] 15
#>
#> $parameterMap$`_KSCut_`$values
#> [1] "_KSCut_"
#>
#> $parameterMap$`_KSCut_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_C_`
#> $parameterMap$`_C_`$parameter
#> [1] "_C_"
#>
#> $parameterMap$`_C_`$type
#> [1] "num"
#>
#> $parameterMap$`_C_`$label
#> [1] "Area Under ROC"
#>
#> $parameterMap$`_C_`$length
#> [1] 8
#>
#> $parameterMap$`_C_`$order
#> [1] 11
#>
#> $parameterMap$`_C_`$values
#> [1] "_C_"
#>
#> $parameterMap$`_C_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_PartInd_`
#> $parameterMap$`_PartInd_`$parameter
#> [1] "_PartInd_"
#>
#> $parameterMap$`_PartInd_`$type
#> [1] "num"
#>
#> $parameterMap$`_PartInd_`$label
#> [1] "Partition Indicator"
#>
#> $parameterMap$`_PartInd_`$length
#> [1] 8
#>
#> $parameterMap$`_PartInd_`$order
#> [1] 2
#>
#> $parameterMap$`_PartInd_`$values
#> [1] "_PartInd_"
#>
#> $parameterMap$`_PartInd_`$preformatted
#> [1] FALSE
#>
#>
#> $parameterMap$`_TargetName_`
#> $parameterMap$`_TargetName_`$label
#> [1] "Target Name"
#>
#> $parameterMap$`_TargetName_`$length
#> [1] 10
#>
#> $parameterMap$`_TargetName_`$order
#> [1] 1
#>
#> $parameterMap$`_TargetName_`$parameter
#> [1] "_TargetName_"
#>
#> $parameterMap$`_TargetName_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_TargetName_`$type
#> [1] "char"
#>
#> $parameterMap$`_TargetName_`$values
#> [1] "_TargetName_"
#>
#>
#>
#> $data
#> dataMap._RASE_ dataMap._NObs_ dataMap._GINI_ dataMap._GAMMA_
#> 1 0.13122576 2000 -0.006248056 NA
#> 2 0.07568684 2000 0.024727905 NA
#> 3 0.65538405 2000 0.007690031 NA
#> dataMap._formattedPartition_ dataMap._DataRole_ dataMap._MCE_ dataMap._ASE_
#> 1 0 VALIDATE 0.4985 0.017220199
#> 2 1 TRAIN 0.4850 0.005728498
#> 3 2 TEST 0.5045 0.429528254
#> dataMap._MCLL_ dataMap._KS_ dataMap._KSPostCutoff_ dataMap._DIV_
#> 1 7.033599e-05 0.01957631 NA 2000
#> 2 3.935217e-05 0.03919829 NA 2000
#> 3 0.000000e+00 0.02007008 NA 2000
#> dataMap._TAU_ dataMap._KSCut_ dataMap._C_ dataMap._PartInd_
#> 1 -0.004418774 0.5 0.496876 0
#> 2 0.017485785 0.5 0.512364 1
#> 3 0.005439022 0.5 0.503845 2
#> dataMap._TargetName_ rowNumber header
#> 1 label 1 NA
#> 2 label 2 NA
#> 3 label 3 NA
#>
#> $version
#> [1] 1
#>
#> $xInteger
#> [1] FALSE
#>
#> $yInteger
#> [1] FALSE
#>
df2 <- data.frame(actual = rnorm(6000, 1000, 100),
predicted = rnorm(6000, 1000, 100),
partition = rep_len(1:3, 6000))
calculateFitStat(targetName = "actual",
targetPredicted = "predicted",
df2[df2$partition == 1, ],
df2[df2$partition == 2, ],
df2[df2$partition == 3, ],
type = "interval",
noFile = TRUE)
#> $creationTimeStamp
#> NULL
#>
#> $modifiedTimeStamp
#> NULL
#>
#> $createdBy
#> NULL
#>
#> $modifiedBy
#> NULL
#>
#> $id
#> NULL
#>
#> $name
#> [1] "dmcas_fitstat"
#>
#> $description
#> NULL
#>
#> $revision
#> [1] 0
#>
#> $groupName
#> NULL
#>
#> $order
#> [1] 0
#>
#> $type
#> NULL
#>
#> $parameterMap
#> $parameterMap$`_ASE_`
#> $parameterMap$`_ASE_`$label
#> [1] "Average Squared Error"
#>
#> $parameterMap$`_ASE_`$length
#> [1] 8
#>
#> $parameterMap$`_ASE_`$order
#> [1] 6
#>
#> $parameterMap$`_ASE_`$parameter
#> [1] "_ASE_"
#>
#> $parameterMap$`_ASE_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_ASE_`$type
#> [1] "num"
#>
#> $parameterMap$`_ASE_`$values
#> [1] "_ASE_"
#>
#>
#> $parameterMap$`_DIV_`
#> $parameterMap$`_DIV_`$label
#> [1] "Divisor for ASE"
#>
#> $parameterMap$`_DIV_`$length
#> [1] 8
#>
#> $parameterMap$`_DIV_`$order
#> [1] 7
#>
#> $parameterMap$`_DIV_`$parameter
#> [1] "_DIV_"
#>
#> $parameterMap$`_DIV_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_DIV_`$type
#> [1] "num"
#>
#> $parameterMap$`_DIV_`$values
#> [1] "_DIV_"
#>
#>
#> $parameterMap$`_DataRole_`
#> $parameterMap$`_DataRole_`$label
#> [1] "Data Role"
#>
#> $parameterMap$`_DataRole_`$length
#> [1] 10
#>
#> $parameterMap$`_DataRole_`$order
#> [1] 2
#>
#> $parameterMap$`_DataRole_`$parameter
#> [1] "_DataRole_"
#>
#> $parameterMap$`_DataRole_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_DataRole_`$type
#> [1] "char"
#>
#> $parameterMap$`_DataRole_`$values
#> [1] "_DataRole_"
#>
#>
#> $parameterMap$`_MAE_`
#> $parameterMap$`_MAE_`$label
#> [1] "Mean Absolute Error"
#>
#> $parameterMap$`_MAE_`$length
#> [1] 8
#>
#> $parameterMap$`_MAE_`$order
#> [1] 9
#>
#> $parameterMap$`_MAE_`$parameter
#> [1] "_MAE_"
#>
#> $parameterMap$`_MAE_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_MAE_`$type
#> [1] "num"
#>
#> $parameterMap$`_MAE_`$values
#> [1] "_MAE_"
#>
#>
#> $parameterMap$`_MSLE_`
#> $parameterMap$`_MSLE_`$label
#> [1] "Mean Squared Logarithmic Error"
#>
#> $parameterMap$`_MSLE_`$length
#> [1] 8
#>
#> $parameterMap$`_MSLE_`$order
#> [1] 11
#>
#> $parameterMap$`_MSLE_`$parameter
#> [1] "_MSLE_"
#>
#> $parameterMap$`_MSLE_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_MSLE_`$type
#> [1] "num"
#>
#> $parameterMap$`_MSLE_`$values
#> [1] "_MSLE_"
#>
#>
#> $parameterMap$`_NObs_`
#> $parameterMap$`_NObs_`$label
#> [1] "Sum of Frequencies"
#>
#> $parameterMap$`_NObs_`$length
#> [1] 8
#>
#> $parameterMap$`_NObs_`$order
#> [1] 5
#>
#> $parameterMap$`_NObs_`$parameter
#> [1] "_NObs_"
#>
#> $parameterMap$`_NObs_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_NObs_`$type
#> [1] "num"
#>
#> $parameterMap$`_NObs_`$values
#> [1] "_NObs_"
#>
#>
#> $parameterMap$`_PartInd_`
#> $parameterMap$`_PartInd_`$label
#> [1] "Partition Indicator"
#>
#> $parameterMap$`_PartInd_`$length
#> [1] 8
#>
#> $parameterMap$`_PartInd_`$order
#> [1] 3
#>
#> $parameterMap$`_PartInd_`$parameter
#> [1] "_PartInd_"
#>
#> $parameterMap$`_PartInd_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_PartInd_`$type
#> [1] "num"
#>
#> $parameterMap$`_PartInd_`$values
#> [1] "_PartInd_"
#>
#>
#> $parameterMap$`_RASE_`
#> $parameterMap$`_RASE_`$label
#> [1] "Root Average Squared Error"
#>
#> $parameterMap$`_RASE_`$length
#> [1] 8
#>
#> $parameterMap$`_RASE_`$order
#> [1] 8
#>
#> $parameterMap$`_RASE_`$parameter
#> [1] "_RASE_"
#>
#> $parameterMap$`_RASE_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_RASE_`$type
#> [1] "num"
#>
#> $parameterMap$`_RASE_`$values
#> [1] "_RASE_"
#>
#>
#> $parameterMap$`_RMAE_`
#> $parameterMap$`_RMAE_`$label
#> [1] "Root Mean Absolute Error"
#>
#> $parameterMap$`_RMAE_`$length
#> [1] 8
#>
#> $parameterMap$`_RMAE_`$order
#> [1] 10
#>
#> $parameterMap$`_RMAE_`$parameter
#> [1] "_RMAE_"
#>
#> $parameterMap$`_RMAE_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_RMAE_`$type
#> [1] "num"
#>
#> $parameterMap$`_RMAE_`$values
#> [1] "_RMAE_"
#>
#>
#> $parameterMap$`_RMSLE_`
#> $parameterMap$`_RMSLE_`$label
#> [1] "Root Mean Squared Logarithmic Error"
#>
#> $parameterMap$`_RMSLE_`$length
#> [1] 8
#>
#> $parameterMap$`_RMSLE_`$order
#> [1] 12
#>
#> $parameterMap$`_RMSLE_`$parameter
#> [1] "_RMSLE_"
#>
#> $parameterMap$`_RMSLE_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_RMSLE_`$type
#> [1] "num"
#>
#> $parameterMap$`_RMSLE_`$values
#> [1] "_RMSLE_"
#>
#>
#> $parameterMap$`_TargetName_`
#> $parameterMap$`_TargetName_`$label
#> [1] "Target Name"
#>
#> $parameterMap$`_TargetName_`$length
#> [1] 3
#>
#> $parameterMap$`_TargetName_`$order
#> [1] 1
#>
#> $parameterMap$`_TargetName_`$parameter
#> [1] "_TargetName_"
#>
#> $parameterMap$`_TargetName_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_TargetName_`$type
#> [1] "char"
#>
#> $parameterMap$`_TargetName_`$values
#> [1] "_TargetName_"
#>
#>
#> $parameterMap$`_formattedPartition_`
#> $parameterMap$`_formattedPartition_`$label
#> [1] "Formatted Partition"
#>
#> $parameterMap$`_formattedPartition_`$length
#> [1] 12
#>
#> $parameterMap$`_formattedPartition_`$order
#> [1] 4
#>
#> $parameterMap$`_formattedPartition_`$parameter
#> [1] "_formattedPartition_"
#>
#> $parameterMap$`_formattedPartition_`$preformatted
#> [1] FALSE
#>
#> $parameterMap$`_formattedPartition_`$type
#> [1] "char"
#>
#> $parameterMap$`_formattedPartition_`$values
#> [1] "_formattedPartition_"
#>
#>
#>
#> $data
#> dataMap._ASE_ dataMap._DIV_ dataMap._DataRole_ dataMap._MAE_ dataMap._MSLE_
#> 1 19619.86 2000 VALIDATE 112.7321 0.02020329
#> 2 19452.11 2000 TRAIN 112.0202 0.01992321
#> 3 20080.16 2000 TEST 113.8536 0.02060902
#> dataMap._NObs_ dataMap._PartInd_ dataMap._RASE_ dataMap._RMAE_
#> 1 2000 0 140.0709 10.61754
#> 2 2000 1 139.4708 10.58396
#> 3 2000 2 141.7045 10.67022
#> dataMap._RMSLE_ dataMap._TargetName_ dataMap._formattedPartition_ rowNumber
#> 1 0.1421383 actual 0 1
#> 2 0.1411496 actual 1 2
#> 3 0.1435584 actual 2 3
#>
#> $xInteger
#> [1] FALSE
#>
#> $yInteger
#> [1] FALSE
#>