Added 30/06/2025
Machine Learning / regression
svr_bennett2006
Datasets
Dimension
{
"x": 19,
"y": 804,
"F": 1,
"G": 11,
"H": 0,
"f": 1,
"g": 2388,
"h": 0
}Solution
{
"optimality": "feasible",
"x": [1,1,1,0.00615244,-0.8459975,-0.08478079,-0.00612273,-0.2105129,0.50659832,0.85530825,-0.49095779,0.01608997,-0.08182366,-0.00441689,-0.00474096,-0.09918633,0.65285064,1.50846709,0.15002266],
"y": [0.01608997,-0.08182366,-0.08478079,-0.00612273,-0.2105129,0.65285064,0.85530825,-0.49095779,0.01227513,-0.8459975,-0.02749269,-0.00474096,-0.14966834,0.56915434,1.50846709,-0.39020805,0.00615244,-0.53477039,-0.00441689,-0.00481153,-0.09918633,0.50659832,1.25382408,0.15002266,-1e-8,-1e-8,-1e-8,-1e-8,2.29084533,2.88807563,2.71074055,3.75486885,2.13247951,1.31648233,2.51223313,0.01641614,1.30300819,1.04111842,-1e-8,3.12318035,1.01235183,0.36850589,-1e-8,0.81956437,1.05047066,0.41097923,5.32149449,7.31489235,5.30499905,2.12027707,4.51943482,2.58190582,1.1141824,-1e-8,0.824996,-1e-8,-1e-8,-1e-8,6.66691508,-1e-8,0.4800348,1.88215262,0.87593778,0.34302431,2.20052707,0.24136777,3.04218517,0.56269141,1.70099625,-1e-8,-1e-8,-1e-8,1.14248376,-1e-8,0.40987013,2.35807199,-1e-8,-1e-8,0.39419074,2.67944342,1.66501343,0.35967189,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,2.74174679,1.17343947,-1e-8,0.44795177,1.59623248,-1e-8,3.79008092,1.80806021,-1e-8,0.91958963,-1e-8,0.36140986,0.82356692,0.34753343,3.35182877,-1e-8,5.16741558,1.3108511,2.31141122,1.78786199,-1e-8,-1e-8,2.57384085,0.28534187,0.49516685,0.29412511,0.0212716,-1e-8,-1e-8,1.33508477,-1e-8,-1e-8,0.77450702,2.01991664,1.2030107,3.24217555,1.59997859,1.1791515,-1e-8,2.52001655,-1e-8,-1e-8,3.70525756,1.29155098,-1e-8,2.69099547,2.66919442,2.06455087,9.58631721,2.21161293,-1e-8,6.88658091,0.52596727,0.72367278,0.16776389,0.04596665,-1e-8,1.3110228,4.12738068,-1e-8,-1e-8,3.22356831,-1e-8,2.69503223,-1e-8,1.61601453,-1e-8,0.08997367,2.62565109,0.48536282,-1e-8,-1e-8,0.98416865,0.31583623,6.15764108,0.04499313,0.19719694,3.59917923,3.13376826,3.58777899,0.993721,1.59326672,-1e-8,1.41519844,-1e-8,4.52193206,1.76757384,2.52915579,0.07607139,0.41770982,0.97936067,0.73861619,-1e-8,1.95632547,0.5617784,-1e-8,-1e-8,0.24669598,-1e-8,3.04980748,-1e-8,1.44246882,3.22272745,2.23542082,0.67504749,2.08173293,2.43346827,-1e-8,4.35408154,0.70433031,1.58490256,0.56461004,4.51690277,8.4677849,2.78629583,-1e-8,2.16549803,-1e-8,-1e-8,-1e-8,3.35388332,1.95440083,0.16016815,-1e-8,5.96982606,-1e-8,12.43843598,1.20187616,6.92819271,10.1349917,11.18670046,8.34610118,3.69882242,8.07253456,-1e-8,3.97872548,6.59287045,4.55734343,4.86289285,-1e-8,1.2178618,1.04014612,1.75521642,2.37508073,-1e-8,1.4206669,4.05680973,0.5797481,0.81841691,1.69087586,2.08930143,-1e-8,1.13227906,0.34727275,-1e-8,-1e-8,-1e-8,-1e-8,2.9613336,0.30019295,1.91139916,3.67456192,0.52508461,0.59568476,-1e-8,3.086753,4.97626393,5.85840551,-1e-8,-1e-8,3.54100006,1.00982621,-1e-8,-1e-8,2.0295325,0.50207201,1.2493489,3.21370907,3.18627998,2.43970879,1.5260267,0.90856896,-1e-8,1.12356293,2.72907802,1.55986036,-1e-8,8.62239092,2.40131138,4.19210021,0.78378564,3.6261749,-1e-8,0.16583244,9.64179416,-1e-8,1.02284257,-1e-8,0.96561653,-1e-8,-1e-8,2.06362596,1.27355117,1.03398505,1.88501591,-1e-8,-1e-8,0.65832848,-1e-8,2.46312764,0.87671141,1.04924807,1.18775821,-1e-8,-1e-8,0.24170318,1.24509792,-1e-8,0.13062552,-1e-8,-1e-8,0.2756815,-1e-8,0.47429582,0.58644901,-1e-8,2.66498717,1.27823213,2.11741869,0.70338681,0.66923389,-1e-8,-1e-8,0.23047788,1.02575876,-1e-8,-1e-8,1.69971006,1.30007979,3.30548095,1.53177646,0.62334407,-1e-8,1.04959669,0.65713599,0.55814544,3.05334015,2.31680985,1.00374789,4.19322182,-1e-8,-1e-8,2.38583603,-1e-8,1.72460363,2.62279575,2.82367373,-1e-8,0.15690286,0.51540876,-1e-8,0.19286002,-1e-8,-1e-8,-1e-8,-1e-8,7.70182453,-1e-8,-1e-8,-1e-8,-1e-8,3.55253461,2.01911692,0.26115103,-1e-8,1.0972859,-1e-8,2.48550631,1.78753118,0.07610515,-1e-8,1.3762943,1.35576816,-1e-8,-1e-8,0.47739116,-1e-8,-1e-8,-1e-8,0.70611837,0.82096547,1.74481539,1.19965199,2.67516061,2.33228422,2.4480018,0.75322729,1.32912178,-1e-8,-1e-8,0.36627633,-1e-8,3.22154211,6.42245705,2.72000528,4.29260227,10.44006519,4.72274356,0.73646073,-1e-8,-1e-8,2.21839324,-1e-8,2.60211756,4.35202027,3.28018292,1.91726194,-1e-8,2.83989267,4.22893115,-1e-8,2.11004074,4.11900769,0.09221893,-1e-8,0.89430195,0.08838518,1.6974141,2.65886353,-1e-8,0.22452672,1.41615823,2.63486101,1.2872849,7.93970036,1.4431017,0.67537299,5.40712669,4.47043171,5.58693902,3.75963284,4.55766831,-1e-8,1.37225979,0.49005815,3.42418312,2.4059898,1.97576009,0.72905861,0.97316175,1.20837598,1.60193613,-1e-8,-1e-8,1.85797528,-1e-8,-1e-8,-1e-8,-1e-8,4.23315112,-1e-8,0.65457537,2.92078826,1.0506923,2.40605328,3.5438695,3.86125774,-1e-8,5.17239689,0.37638944,1.82663589,0.56390589,4.51684881,8.97613894,3.51749637,0.92560154,2.67437009,-1e-8,-1e-8,0.10038547,2.93420317,2.2166368,-1e-8,-1e-8,4.56955578,-1e-8,12.73029983,0.03510514,7.1371438,11.87250635,12.45708268,8.79186229,2.94564405,8.87762015,0.03115921,2.32602823,4.17766758,7.19556322,4.25909682,-1e-8,0.82874641,0.83294159,1.84482674,1.93824277,-1e-8,2.72191388,6.2119592,-1e-8,0.44720546,2.15789372,2.76758351,-1e-8,3.03866252,2.26817967,0.02990833,-1e-8,-1e-8,-1e-8,0.94852283,-1e-8,0.59091551,4.27223745,1.99917741,2.49732309,-1e-8,4.24242731,3.91278179,5.05688396,-1e-8,0.06042008,3.68824403,1.18973549,-1e-8,0.12357457,2.55670751,1.50920278,1.65501262,2.69554109,5.31288824,3.98040066,0.93514438,0.95065914,-1e-8,1.86412373,2.03110842,2.18595449,-1e-8,10.24621796,2.36494779,3.84083523,-1e-8,4.42907969,-1e-8,0.19697024,11.10562389,-1e-8,1.14765806,-1e-8,0.74194873,-1e-8,-1e-8,1.52628431,0.43300828,0.24538073,0.88974507,-1e-8,0.05774464,0.81440083,-1e-8,3.55289045,-1e-8,1.71275965,0.56843913,-1e-8,0.24435901,-1e-8,2.31919142,-1e-8,0.96241746,0.76297286,-1e-8,-1e-8,0.04994866,-1e-8,-1e-8,-1e-8,3.73361859,-1e-8,1.18639713,-1e-8,-1e-8,0.11192252,-1e-8,-1e-8,0.83711583,-1e-8,-1e-8,1.32625189,1.22147536,3.1393017,0.67441754,-1e-8,0.01910145,-1e-8,-1e-8,1.80024826,4.24931087,3.68120481,2.52475891,5.66724326,-1e-8,0.95592017,1.02506607,-1e-8,0.17588132,1.68337441,1.763555,-1e-8,-1e-8,0.33997575,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,6.23462084,-1e-8,-1e-8,-1e-8,-1e-8,2.69301016,0.53533064,-1e-8,0.03825031,-1e-8,1.25701711,1.06145743,2.91050809,-1e-8,0.47107519,1.50659831,1.14247292,-1e-8,-1e-8,0.09511224,-1e-8,-1e-8,-1e-8,0.00882779,-1e-8,2.0733885,0.4440761,1.71578229,1.40564968,1.15412705,1.58417631,-1e-8,-1e-8,0.01446564,0.11090224,-1e-8,2.25543432,5.11217983,1.30315741,2.92016844,8.76331207,3.67202923,0.37521855,-1e-8,-1e-8,0.97353121,1.21505024,0.96201872,2.98773539,2.41317145,1.95587274,-1e-8,2.26049724,4.74518947,-1e-8,1.12395521,3.04249834,1.81621857,1.27934074,2.50646285,0.63395449,-1e-8,0.52415049,-1e-8,0.45951166,-1e-8,-1e-8,1.04921356,-1e-8,2.07015154,0.21455962,-1e-8,0.07218693,3.02011843,1.68720381,1.54922989,-1e-8,1.10685687,-1e-8,3.01306004,-1e-8,0.03438064,2.81411754,4.28125561,1.77018975,0.11832725,1.66595166,0.11993381,-1e-8,-1e-8,1.46003277,-1e-8,-1e-8,-1e-8,5.97084996,-1e-8,-1e-8,0.56800367,-1e-8,0.82793039,0.42098375,1.13948454,2.01792784,-1e-8,0.08396251,0.3513395,-1e-8,0.21690322,-1e-8,2.27938821,1.09447152,1.01813678,-1e-8,-1e-8,-1e-8,1.1847244,-1e-8,0.02550969,0.0651226,0.44383062,0.80021959,1.66858315,0.36269846,1.93387822,-1e-8,-1e-8,1.76287126,0.68317774,-1e-8,-1e-8,2.23569836,-1e-8,1.77344638,-1e-8,0.88093881,1.65581921,2.65442131,1.46379264,4.05193807,0.91765744,0.78917432,2.43908186,2.80259151,3.46520486,1.09144603,2.23439371,5.86938021,-1e-8,2.53049204,-1e-8,0.1718106,-1e-8,-1e-8,1.19231513,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,-1e-8,1.34270311,0.58182776,2.45709943,5.55180888,-1e-8,1.65684309,2.19810973,3.50569101,4.39522155,13.40842091,6.09817387,1.73780241,8.70387007,3.80546414,-1e-8,-1e-8,-1e-8,-1e-8,0.03521679,1.53162311,-1e-8,-1e-8,1.18873173,-1e-8,0.33714501,-1e-8,1.76217439],
"F": 9.337109,
"f": 1202.428702
}Dimension
{
"x": 13,
"y": 19149,
"F": 1,
"G": 8,
"H": 0,
"f": 1,
"g": 57432,
"h": 0
}Solution
{
"optimality": "unknown"
}Dimension
{
"x": 21,
"y": 1563,
"F": 1,
"G": 12,
"H": 0,
"f": 1,
"g": 4662,
"h": 0
}Solution
{
"optimality": "unknown"
}Dimension
{
"x": 9,
"y": 189,
"F": 1,
"G": 6,
"H": 0,
"f": 1,
"g": 558,
"h": 0
}Solution
{
"optimality": "global",
"x": [3.5391,0,0.8209,3,-5,7,3,-5,7],
"y": [3,-5,7,3,-5,7,3,-5,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
"F": 0,
"G": [3.5391,0,0.8209,0,0,0],
"H": [],
"f": 328.9041,
"g": [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],
"h": []
}$title svr_bennett2006
$onText
{description}
$offText
set i / 1*4 /;
variables obj_val_upper, obj_val_lower, x(i), y(i);
equations obj_eq_upper, obj_eq_lower
G1_upper, G2_upper, H1_upper, H2_upper,
g1_lower, g2_lower, h1_lower, h2_lower;
* Objective functions
obj_eq_upper.. obj_val_upper =e= ;
obj_eq_lower.. obj_val_lower =e= ;
* Upper-level constraints
G1_upper.. x('1') =g= 1.0;
G2_upper.. x('2') =g= -2.0;
H1_upper.. x('3') =e= 3.0;
H2_upper.. x('4') =e= -4.0;
* Lower-level constraints
g1_lower.. y('1') =g= 1.0;
g2_lower.. y('2') =g= -2.0;
h1_lower.. y('3') =e= 3.0;
h2_lower.. y('4') =e= -4.0;
* Solve
model svr_bennett2006 / all /;
$echo bilevel x min obj_val_lower y obj_eq_lower g1_lower g2_lower h1_lower h2_lower > "%emp.info%"
solve svr_bennett2006 us emp min obj_val_upper;\subsection{svr\_bennett2006}
\label{subsec:svr_bennett2006}
% Bold characters for w, w lower bound and w upper bound
\newcommand{\w}{\mathbf{w}}
\newcommand{\wlb}{\mathbf{\bar{w}}}
\newcommand{\wub}{\mathbf{\munderbar{w}}}
% Cite Bennett 2006
This bilevel program was first introduced by Bennett et. al. in 2006 \cite[Section~IV, Equation~1]{Bennett2006} though it has been presented in many forms since then.
% Introduce the hyperparameters
The classic Support Vector Regression \eqref{eq:svr} approach has two hyperparameters, the regularisation constant $C$ and the tube width $\epsilon$. A third hyperparameter $\lambda$ controls the multitask learning
learning \cite{Caruana1997}. Then $\wlb$ and $\wub$ are lower and upper bounds, respectively, on the model weights providing feature selection.
% T-fold cross-validation split
The data consists of feature vectors $x_i\in\R^d$ and labels $y_i\in\R$ for $i\in\Omega$. The data indices are split into $T$ distinct partitions $\Omega_t$ for $t=1,\dots,T$. T-fold cross-validation methodology is used such that $\Omega_t$ is used to evaluate the validation loss in the upper-level, while its complement $\bar{\Omega}_t=\Omega\setminus\Omega_t$ is used to evaluate training loss in the lower-level.
% Upper-level description
The upper-level program seeks to choose optimal hyperparameters $C,\epsilon,\lambda,\wub,\wlb$ such that the optimal regression weights $\w^t$ chosen by the lower-level \eqref{eq:svr} program result in the minimum average validation loss.
% Upper-level equation
\begin{equation*}
\begin{aligned}
\minimise_{C, \epsilon, \lambda, \wlb, \wub, \w^t} \quad
&\frac{1}{T}\sum_{t=1}^{T} \frac{1}{|\Omega_t|} \sum_{i\in\Omega_t} |x_i^\top\w^t-y_i|&
\\
\subjectto \quad
& C, \epsilon, \lambda \geq 0,& \\
& \wlb \leq \wub,& \\
&\w^t \text{ solve \eqref{eq:svr}}.
\end{aligned}
\end{equation*}
% Lower-level description
The lower-level program seeks to minimise the sum of $\epsilon$-insensitive residuals over training data $\max\{||x_j^\top\w - y_i - \epsilon, 0\}$. The slack variable $\{e_j\}_{j\in\bar{\Omega}_t}$ are introduced to remove the max operator. Two extra terms are added for regularisation.
% Lower-level equation
\begin{align}
\label{eq:svr}
\tag{SVR}
\begin{aligned}
&\minimise_{\w} \quad &
&C\sum_{j\in\bar{\Omega}_t}e_j + \frac{1}{2}\|\w\|_2^2+\frac{\lambda}{2}\|\w-\w_0\|_2^2&
\\
&\subjectto &
& \wlb\leq\w\leq\wub& \\
&&&\left\{
\begin{array}{l}
e_j \geq \phantom{-}x_j^\top\w - y_j - \epsilon\\
e_j \geq -x_j^\top\w + y_j - \epsilon\\
e_j \geq 0
\end{array}
\right\}\quad \text{ for } j\in\bar{\Omega}_t&
\end{aligned}
\end{align}
Since the lower-level program is convex and has a Slater's point, Bennett et. al. suggest solving this with a KKT reformulation.classdef svr_bennett2006
%{
Kristin P Bennett, Jing Hu, Xiaoyun Ji, Gautam Kunapuli, and Jong-Shi Pang.
Model selection via bilevel optimization.
In the 2006 IEEE International Joint Conference on Neural Network Proceedings, pages 1922–1929.
%}
properties(Constant)
name = 'svr_bennett2006';
category = 'machine_learning';
subcategory = 'regression';
T_folds = 3;
datasets = {
'regression_auto_mpg.csv';
'regression_combined_cycle_power_plant.csv';
'regression_energy_efficiency.csv';
'regression_toy.csv';
};
paths = fullfile('bolib3', 'data', 'regression', svr_bennett2006.datasets);
end
methods(Static)
% //==================================================\\
% || F ||
% \\==================================================//
% Upper-level objective function (convex)
% Computes sum over folds of sum_i | Xw - y |
function obj = F(~, y, data)
obj = 0;
d = data.d;
for t = 1:svr_bennett2006.T_folds
w_t = y((t-1)*d + 1 : t*d);
residual = data.val_features{t} * w_t - data.val_labels{t};
obj = obj + (sum(abs(residual))/length(data.val_labels{t}));
end
end
% //==================================================\\
% || G ||
% \\==================================================//
% Upper-level inequality constraints (linear)
% C >= 0
% epsilon >= 0
% lambda >= 0
% w_ub - w_lb >= 0
function val = G(x, ~, data)
d = data.d;
% C, epsilon, lambda >= 0
hp_nonnegativity = x(1:3);
% w_lb <= w_ub => w_ub - w_lb >= 0
w_lb = x(4 : 3 + d);
w_ub = x(4 + d : 3 + 2*d);
w_bounds = w_ub - w_lb;
% Stack them into one vector
val = [hp_nonnegativity; w_bounds];
end
% //==================================================\\
% || H ||
% \\==================================================//
% Upper-level equality constraints (none)
function val = H(~, ~, ~)
val = [];
end
% //==================================================\\
% || f ||
% \\==================================================//
% Lower-level objective function (quadratic)
% obj = sum_t [ C * sum(e_t) + 0.5 * w_t'w_t + lambda * w_t'w_t ]
function obj = f(x, y, data)
% Split variables
[hp_C, ~, hp_lambda, ~, ~, w, e] = svr_bennett2006.variable_split(x, y, data);
obj = 0;
for t = 1:svr_bennett2006.T_folds
obj = obj + hp_C * sum(e{t}) + 0.5 * (w{t}' * w{t}) + hp_lambda * (w{t}' * w{t});
end
end
% //==================================================\\
% || g ||
% \\==================================================//
% Lower-level inequality constraints (linear)
% For each fold t=1,...,T:
% e_j >= + w*x_j - y_j - epsilon
% e_j >= - w*x_j + y_j - epsilon
% e_j >= 0
% w >= w_lb
% w <= w_ub
function constraints = g(x, y, data)
% Split variables
[~, hp_epsilon, ~, w_lb, w_ub, w, e] = svr_bennett2006.variable_split(x, y, data);
% Store all constraints in a cell array
cells = cell(1, 5*svr_bennett2006.T_folds);
for t = 1:svr_bennett2006.T_folds
cells{5*(t-1) + 1} = e{t} + hp_epsilon - (data.train_features{t} * w{t}) + data.train_labels{t};
cells{5*(t-1) + 2} = e{t} + hp_epsilon + (data.train_features{t} * w{t}) - data.train_labels{t};
cells{5*(t-1) + 3} = e{t};
cells{5*(t-1) + 4} = w{t} - w_lb;
cells{5*(t-1) + 5} = w_ub - w{t};
end
% Concatenate all constraints into a single vector
constraints = vertcat(cells{:});
end
% //==================================================\\
% || h ||
% \\==================================================//
% Lower-level equality constraints
function val = h(~, ~, ~)
val = [];
end
% //==================================================\\
% || Variable Split ||
% \\==================================================//
% Upper-level variables:
% x = [C, epsilon, lambda, w_lb, w_ub]
% Lower-level variables:
% y = [w[0], ..., w[T], e[0], ..., e[T]]
%
function [hp_C, hp_epsilon, hp_lambda, w_lb, w_ub, w, e] = variable_split(x, y, data)
% Dimensions
d = data.d;
n_train = data.n_train;
% Upper-level decision variables
hp_C = x(1);
hp_epsilon = x(2);
hp_lambda = x(3);
w_lb = x(4 : 3 + d);
w_ub = x(4 + d : 3 + 2*d);
% Lower-level decision variables
w = cell(1, svr_bennett2006.T_folds);
e = cell(1, svr_bennett2006.T_folds);
for t = 1:svr_bennett2006.T_folds
idx_w_start = (t-1)*d + 1;
idx_w_end = t*d;
w{t} = y(idx_w_start : idx_w_end);
idx_e_start = svr_bennett2006.T_folds*d + (t-1)*n_train + 1;
idx_e_end = svr_bennett2006.T_folds*d + t*n_train;
e{t} = y(idx_e_start : idx_e_end);
end
end
% //==================================================\\
% || Evaluate ||
% \\==================================================//
% Calculates the R2 and RMSE scores of the regression models represented by each fold
function evaluate(~, y, data)
d = data.d;
% Print table headers
fprintf('| %10s | %4s | %10s | %10s | %10s |\n', '', 'fold', 'data', 'R2 score', 'RMSE');
fprintf('| %s | %s | %s | %s | %s |\n', repmat('-',1,10), repmat('-',1,4), repmat('-',1,10), repmat('-',1,10), repmat('-',1,10));
% Evaluate for training and validation folds
tasks = {'Training', 'Validation'};
features_list = {data.train_features, data.val_features};
labels_list = {data.train_labels, data.val_labels};
for idx = 1:2
for t = 1:svr_bennett2006.T_folds
w_t = y((t-1)*d + 1 : t*d);
features = features_list{idx}{t};
labels = labels_list{idx}{t};
predictions = features * w_t;
% RMSE
rmse = sqrt(mean((labels - predictions).^2));
% R2 score
ss_res = sum((labels - predictions).^2);
ss_tot = sum((labels - mean(labels)).^2);
r2 = 1 - ss_res / ss_tot;
% Display
fprintf('| %-10s | %4d | %10s | %10.4f | %10.4f |\n', ...
tasks{idx}, t, sprintf('%dx%d', size(features,1), size(features,2)), r2, rmse);
end
end
end
% //==================================================\\
% || Read Data ||
% \\==================================================//
% The CSV file should have:
% - First row: headers (e.g. 'label', 'feature1', 'feature2', ...)
% - First column: labels (independent variable)
% - Remaining columns: features (dependent variables)
%
% Splits the data into T training and validation folds.
% If len(data) is not divisible by T then the last (len(data)%T) examples are dropped.
% For example consider the dataset [1,...,17] and parameter T is set to 3.
% The procedure can be visualized with the following table:
%
% | | i=1,2,3,4,5 | i=6,7,8,9,10 | i=11,12,13,14,15 |
% |-----|------------------|------------------|------------------|
% | t=1 | validation | train | train |
% | t=2 | train | validation | train |
% | t=3 | train | train | validation |
function data = read_data(filepath)
% Read the csv
options = detectImportOptions(filepath);
data_table = readtable(filepath, options);
% Add a column of ones for the constant term
data_table.constant = ones(height(data_table),1);
% Total number of folds
T = svr_bennett2006.T_folds;
% Extract labels and features
labels = table2array(data_table(:,1));
features = table2array(data_table(:,2:end));
% dimention (total number must be divsible by T)
n_val = floor(length(labels) / T);
n_total = n_val * T;
n_train = n_total - n_val;
% Preallocate cells
val_features = cell(1, T);
val_labels = cell(1, T);
train_features = cell(1, T);
train_labels = cell(1, T);
% Form validation and training sets for each fold t=1,...,T
for t = 1:T
val_indices_t = ((t-1)*n_val + 1):t*n_val;
train_indices_t = setdiff(1:n_total, val_indices_t);
val_features{t} = features(val_indices_t, :);
val_labels{t} = labels(val_indices_t);
train_features{t} = features(train_indices_t, :);
train_labels{t} = labels(train_indices_t);
end
% Form output struct
data = struct();
data.file = filepath;
data.d = size(features, 2);
data.n_val = n_val;
data.val_features = val_features;
data.val_labels = val_labels;
data.n_train = n_train;
data.train_features = train_features;
data.train_labels = train_labels;
end
% Key are the function/variable names
% Values are their dimention
function n = dimention(key, data)
n = dictionary( ...
'x', 3 + 2*data.d, ...
'y', svr_bennett2006.T_folds*(data.d + data.n_train), ...
'F', 1, ...
'G', 3 + data.d, ...
'H', 0, ...
'f', 1, ...
'g', svr_bennett2006.T_folds*(2*data.d + 3*data.n_train), ...
'h', 0 ...
);
if isKey(n,key)
n = n(key);
end
end
end
properties(Constant)
x0 = [
942.69; % C
1.10; % Epsilon
394.61; % Lambda
0.00; -0.81; -0.04; -0.01; -0.38; 0.57; 1.29; -3.18; % w lower bound
0.02; -0.33; -0.01; -0.00; 0.09; 0.64; 1.37; -3.18 % w upper bound
];
y0 = [
0.000349; -0.594078; 0.015399; 0.004389; -0.18728; 0.591777; 1.29; -3.18; % w fold t=0
0.02; -0.752486; 0.01; 0.006345; 0.059886; 0.577763; 1.29; -3.18; % w fold t=1
0.002926; -0.33; 0.04; 0.004406; -0.362387; 0.63628; 1.37; -3.18; % w fold t=2
0.0; 4.296325; 1.252274; 3.325708; 0.0; 0.0; 0.646209; 2.519647; 0.0; 0.024924; 0.252751; 0.0; 1.281405; 0.0;
0.581238; 0.0; 0.038059; 0.855709; 2.960408; 0.377168; 4.414534; 4.779002; 0.0; 0.0; 2.371249; 0.0; 0.0; 1.435884;
0.984823; 0.0; 1.784312; 0.376163; 3.132948; 1.021712; 1.745304; 1.584691; 0.003383; 1.570463; 0.0; 0.953865; 0.0;
1.152161; 4.249906; 2.193958; 0.53174; 0.0; 0.126176; 0.0; 2.777644; 1.579941; 3.512486; 0.0; 1.456302; 5.110235;
1.813491; 0.0; 2.146999; 2.270538; 0.0; 0.0; 3.080307; 7.047226; 4.250518; 1.386629; 0.0; 1.9534; 0.0; 0.0;
1.815792; 0.330653; 0.0; 1.761197; 1.340089; 3.964859; 0.022587; 4.675208; 9.190773; 0.1355; 0.0; 0.0; 0.45286;
2.310917; 1.220063; 0.0; 0.95974; 6.973211; 2.932587; 2.493241; 4.012014; 1.297346; 0.694085; 0.0; 0.0; 0.0;
0.131152; 3.808762; 0.228293; 0.0; 4.630662; 0.0; 9.796482; 1.862521; 0.281683; 2.812953; 0.368985; 0.0; 3.262028;
3.473138; 13.427186; 2.609227; 0.253462; 1.157014; 0.0; 4.722483; 0.271294; 0.32009; 0.0; 0.0; 4.427369; 0.484942;
0.0; 0.0; 0.583326; 2.277108; 1.244839; 5.557292; 0.0; 0.062551; 4.183039; 0.0; 1.174414; 1.050416; 0.0; 0.126927;
1.497728; 1.782488; 0.0; 0.0; 0.420509; 0.58596; 9.8e-05; 0.0; 2.014015; 0.410163; 1.873457; 0.0; 1.568642;
0.441107; 2.063639; 0.0; 3.664282; 1.529542; 0.0; 3.346912; 13.327642; 3.667618; 0.0; 0.98113; 3.969028; 0.0; 0.0;
0.294799; 3.886231; 1.637845; 0.045364; 4.645961; 1.265545; 2.849633; 0.0; 0.0; 1.436774; 4.233925; 1.059105;
2.92423; 1.779308; 0.468137; 0.379592; 0.003632; 2.133575; 2.626549; 0.0; 0.214933; 2.053737; 0.0; 0.007523;
0.315526; 0.0; 2.118025; 0.53898; 0.0; 0.0; 2.00817; 0.0; 0.0; 2.062082; 1.527982; 2.951778; 0.0; 0.0; 0.271359;
0.964612; 2.607597; 2.080161; 0.0; 5.699111; 0.0; 1.825959; 5.0573; 3.484852; 0.410585; 0.0; 0.215467; 0.0; 0.0;
0.122682; 0.345693; 0.576469; 0.406792; 1.008657; 0.914937; 1.049381; 3.838211; 0.974269; 1.220562; 1.821084;
0.676337; 0.0; 1.200291; 1.029445; 0.0; 0.091608; 0.178226; 4.68604; 2.298553; 0.562112; 0.0; 0.0; 0.178522; 0.0;
1.927432; 0.11813; 0.248471; 1.175662; 0.518371; 1.050619; 2.040495; 0.0; 7.192875; 0.0; 0.420858; 12.263202;
0.380994; 2.078746; 0.39583; 5.609143; 0.0; 0.456673; 12.755954; 0.019168; 1.635657; 0.609954; 1.962734; 0.0;
5.555875; 0.060674; 2.56366; 3.279727; 3.354162; 0.0; 6.494664; 0.152595; 1.265725; 0.0; 0.0; 0.0; 0.0; 3.231914;
0.0; 0.0; 0.419252; 6.684132; 1.342993; 3.38836; 0.0; 9.311952; 0.425559; 1.437698; 0.0; 1.936583; 0.240557;
2.150241; 0.0; 1.695011; 1.427797; 0.0; 0.0; 0.54037; 0.0; 0.0; 0.572981; 0.0; 9.360465; 0.0; 0.85105; 0.497131;
0.0; 0.0; 0.501531; 0.0; 3.514277; 0.202377; 0.385678; 0.0; 0.960654; 2.365251; 4.652084; 3.822686; 0.0; 0.0;
3.043464; 0.898241; 2.647794; 0.0; 1.710612; 0.0; 0.159314; 0.56644; 0.100085; 1.641978; 6.986741; 1.473045;
0.139476; 0.951288; 0.0; 0.585494; 0.0; 0.0; 0.565002; 0.0; 0.566856; 0.0; 4.124855; 0.261632; 1.963646; 0.993579;
1.136538; 2.192689; 1.646623; 0.0; 4.370751; 0.0; 0.778449; 3.867727; 0.625887; 9.650021; 1.577135; 0.0; 0.0; 0.0;
6.574404; 0.764196; 1.595786; 0.0; 0.0; 3.392697; 1.939825; 0.700495; 0.052874; 9.754728; 0.0; 0.0; 3.48768;
0.802873; 0.0; 2.043519; 0.0; 0.0; 3.921929; 1.959594; 0.0; 0.0; 0.3127; 0.0; 0.0; 2.983815; 0.0; 1.83189; 4.96362;
5.83562; 0.0; 0.941767; 2.441303; 0.0; 0.334927; 1.322773; 2.651246; 0.0; 0.0; 0.0; 0.099974; 0.512102; 0.214042;
0.711576; 1.192063; 2.142727; 0.0; 1.159921; 2.395009; 2.678757; 0.375006; 2.600225; 0.117835; 0.0; 2.957041;
11.764845; 4.778022; 0.0; 0.064488; 2.119516; 0.0; 0.0; 0.202364; 4.43261; 1.040206; 0.081288; 4.686763; 2.584367;
2.381216; 0.0; 0.031196; 0.537794; 4.35308; 2.031881; 1.740447; 1.296947; 0.0; 0.510216; 0.354055; 2.283879;
3.607174; 0.0; 0.322203; 0.986396; 0.025445; 0.277725; 0.395995; 0.0; 2.941097; 0.98901; 0.181441; 0.0; 1.41994;
0.0; 0.0; 1.872985; 1.565112; 2.497949; 0.0; 0.0; 0.312136; 1.431995; 2.096221; 1.882741; 0.0; 4.752209; 0.0;
1.173437; 3.60508; 2.644296; 0.0; 0.0; 0.132417; 0.0; 0.0; 0.788642; 0.0; 0.0; 0.613296; 2.222431; 0.0; 1.664929;
3.181359; 1.867732; 1.063342; 1.870465; 1.469997; 0.0; 0.245035; 2.612102; 0.0; 0.642298; 0.0; 4.092816; 1.507542;
0.483124; 0.0; 0.0; 0.0; 0.0; 1.618722; 0.0; 1.212049; 1.579186; 0.0; 0.0; 2.884045; 0.0; 6.386427; 0.0; 0.227493;
10.531423; 0.873409; 2.440361; 0.0; 3.157825; 0.0; 1.182512; 11.198405; 0.0; 2.27279; 0.298399; 1.624341; 0.0;
4.756753; 0.556705; 2.043668; 4.110234; 4.228567; 0.0; 4.796603; 0.0; 1.350014; 0.0; 0.0; 0.0; 0.0; 2.920823; 0.0;
0.0; 0.883152; 6.584653; 0.025556; 0.716609; 0.0; 8.783497; 0.641825; 1.524132; 0.0; 0.873738; 0.0; 2.6162;
0.200578; 0.680832; 1.510734; 0.0; 0.0; 2.202807; 0.0; 0.0; 2.725498; 0.0; 8.552593; 0.0; 0.298605; 0.495948;
0.0; 0.0; 0.197676; 0.0; 2.514259; 0.595785; 0.0; 0.626791; 0.0; 2.258893; 4.121462; 3.835965; 0.0; 0.672519;
1.712922; 1.469087; 2.584109; 0.0; 3.22375; 0.0; 0.0; 2.200228; 0.0; 2.392011; 7.79248; 1.767081; 0.306815;
1.572941; 0.0; 1.054235; 0.0; 1.429689; 0.099582; 0.0; 1.004675; 0.0; 2.93243; 0.635162; 1.19989; 3.597754;
2.935238; 1.326419; 1.187039; 0.0; 5.292264; 0.0; 0.71673; 4.494548; 0.511313; 10.591884; 1.498288; 0.0; 0.0; 0.0;
7.471386; 1.550287; 1.511363; 0.0; 0.0; 2.562011; 0.845157; 1.171172; 0.0; 12.579813; 0.0; 0.0; 4.535857; 0.030698;
0.0; 2.078368; 1.421113; 0.0; 2.404634; 2.800321; 0.515842; 0.001106; 0.0; 0.0; 0.0; 3.005475; 0.0; 0.0; 4.466236;
6.003285; 0.0; 0.0; 4.703087; 1.093062; 3.635723; 0.0; 0.0; 0.554857; 2.906585; 0.0; 0.0; 0.26434; 0.0; 1.868009;
0.0; 0.767943; 0.0; 0.0; 0.311107; 3.068584; 0.039162; 4.032188; 4.318037; 0.0; 0.0; 2.085508; 0.0; 0.0; 1.32544;
1.44314; 0.098778; 1.691968; 0.910304; 3.66684; 0.769006; 2.497978; 1.103729; 0.0; 0.770869; 0.0; 1.275256; 0.0;
1.212942; 3.999437; 2.936755; 0.0; 0.0; 0.216068; 0.0; 2.618757; 2.350003; 2.97163; 1.477464; 1.003837; 4.87839;
1.769495; 0.0; 2.109468; 2.59619; 0.0; 0.0; 3.734434; 7.18856; 4.173406; 1.484233; 0.0; 2.099632; 0.0; 0.0;
1.082505; 0.699181; 0.0; 2.296291; 2.060906; 3.294362; 0.0; 4.71305; 9.021503; 0.0; 0.0; 0.075124; 0.149576;
2.21331; 0.96611; 0.0; 0.101291; 7.055904; 3.12131; 1.621537; 3.487494; 0.867704; 0.862871; 0.0; 0.0; 0.0; 0.0;
3.833386; 0.0; 0.046158; 4.818383; 0.0; 9.643603; 1.633465; 0.0; 2.952384; 0.513767; 0.0; 2.650039; 3.054194;
13.118452; 2.230619; 0.484316; 1.363891; 0.419163; 3.972292; 0.0; 0.287555; 0.0; 0.0; 4.473218; 0.439954; 0.0; 0.0;
0.0; 1.991387; 1.142356; 4.898578; 0.0; 0.0; 3.62821; 0.0
];
end
endimport math
import os
from bolib3 import np
import pandas as pd
"""
Kristin P Bennett, Jing Hu, Xiaoyun Ji, Gautam Kunapuli, and Jong-Shi Pang.
Model selection via bilevel optimization.
In the 2006 IEEE International Joint Conference on Neural Network Proceedings, pages 1922–1929.
"""
# Properties
name: str = "svr_bennett2006"
category: str = "machine_learning"
subcategory: str = "regression"
T_folds: int = 3
datasets: list = [
"regression_auto_mpg.csv",
"regression_combined_cycle_power_plant.csv",
"regression_energy_efficiency.csv",
"regression_toy.csv",
]
paths: list = [
os.path.join("bolib3", "data", "regression", dataset) for dataset in datasets
]
# //==================================================\\
# || F ||
# \\==================================================//
def F(x, y, data):
"""
Upper-level objective function (convex)
sum_{t} sum_{i} | (val_features_{ti} * w_{t} - val_labels_{ti}) |
"""
obj = 0
for t in range(T_folds):
w_t = y[t*data['d']: (t + 1)*data['d']]
residuals = np.abs(np.dot(data['val_features'][t], w_t) - data['val_labels'][t])
obj += np.sum(residuals)/len(data['val_labels'][t])
return obj
# //==================================================\\
# || G ||
# \\==================================================//
def G(x, y, data=None):
"""
Upper-level inequality constraints (linear)
C >= 0
epsilon >= 0
lambda >=0
w_lb <= w_ub
"""
d = data['d']
return np.concatenate([
x[0:3],
x[3 + d:3 + 2*d] - x[3:3 + d]
])
# //==================================================\\
# || H ||
# \\==================================================//
def H(x, y, data=None):
"""
Upper-level equality constraints (none)
"""
return np.empty(0)
# //==================================================\\
# || f ||
# \\==================================================//
def f(x, y, data=None):
"""
Lower-level objective function (quadratic)
"""
hp_C, hp_epsilon, hp_lambda, w_lb, w_ub, w, e = variable_split(x, y, data)
obj = 0
for t in range(T_folds):
obj += hp_C*np.sum(e[t]) + 0.5*np.dot(w[t], w[t]) + hp_lambda*np.dot(w[t], w[t])
return obj
# //==================================================\\
# || g ||
# \\==================================================//
def g(x, y, data=None):
"""
Lower-level inequality constraints (linear)
e_j >= + w * x_j - y_j - epsilon
e_j >= - w * x_j + y_j - epsilon
e_j >= 0
"""
hp_C, hp_epsilon, hp_lambda, w_lb, w_ub, w, e = variable_split(x, y, data)
constraints = []
for t in range(T_folds):
constraints.append(e[t] + hp_epsilon - np.dot(data['train_features'][t], w[t]) + data['train_labels'][t])
constraints.append(e[t] + hp_epsilon + np.dot(data['train_features'][t], w[t]) - data['train_labels'][t])
constraints.append(e[t])
constraints.append(w[t] - w_lb)
constraints.append(w_ub - w[t])
return np.concatenate(constraints)
# //==================================================\\
# || h ||
# \\==================================================//
def h(x, y, data=None):
"""
Lower-level equality constraints (none)
"""
return np.empty(0)
# //==================================================\\
# || Variable Split ||
# \\==================================================//
def variable_split(x, y, data):
"""
Splits the vectors x, y into their more meaningful components.
Upper-level variables: x = (C, epsilon, lambda, w_lb, w_ub).
Lower-level variables: y = (w[0], ..., w[T], e[0], ..., e[T]).
"""
# Dimensions of the data
d = data['d']
n_train = data['n_train']
# Upper-level decision variables
hp_C = x[0] # C regularisation hyperparameter
hp_epsilon = x[1] # epsilon tube width hyperparameter
hp_lambda = x[2] # lambda multi task learning hyperparameter
w_lb = x[3:3 + d] # w lower bound
w_ub = x[3 + d:3 + 2*d] # w upper bound
# Lower-level decision variables
w = [None]*T_folds
e = [None]*T_folds
for t in range(0, T_folds):
w[t] = y[t*d: (t + 1)*d]
e[t] = y[T_folds*d + t*n_train: T_folds*d + (t + 1)*n_train]
return hp_C, hp_epsilon, hp_lambda, w_lb, w_ub, w, e
# //==================================================\\
# || Evaluate ||
# \\==================================================//
def evaluate(x, y, data):
"""
Calculates the R2 and RMSE scores of the regression models represented by each fold
"""
# Print table headers
print(f"| {'':^10} | {'fold':^4} | {'data':^10} | {'R2 score':^10} | {'RMSE':^10} |")
print(f"| {'':-^10} | {'':-^4} | {'':-^10} | {'':-^10} | {'':-^10} |")
# Iterate over first training and then validation T-folds
for task, features, observations in (
('Training', data['train_features'], data['train_labels']),
('Validation', data['val_features'], data['val_labels'])
):
rmse_sum = 0
r2_sum = 0
for t in range(T_folds):
# Make predictions on the data
w_t = y[t*data['d']: (t + 1)*data['d']]
predictions_t = np.dot(features[t], w_t)
# Score RMSE and R2
rmse = np.sqrt(np.mean((observations[t] - predictions_t)**2))
r2 = 1 - (
np.sum((observations[t] - predictions_t)**2)/
np.sum((observations[t] - np.mean(observations[t]))**2)
)
# Output results for fold t
print(f"| {task:<10} | {t:^4} | {str(features[t].shape):^10} | {r2:^10.4f} | {rmse:^10.4f} |")
# Add them to the sum
rmse_sum += rmse
r2_sum += r2
# Output mean results
print(f"| {task:<10} | {'mean':^4} | {'':^10} | {r2_sum/T_folds:^10.4f} | {rmse_sum/T_folds:^10.4f} |")
# //==================================================\\
# || Read Data ||
# \\==================================================//
def read_data(filepath=datasets[0]):
"""
The CSV file should have:
- First row: headers (e.g. 'label', 'feature1', 'feature2', ...)
- First column: labels (independent variable)
- Remaining columns: features (dependent variables)
Splits the data into T training and validation folds.
If len(data) is not divisible by T then the last (len(data)%T) examples are dropped.
For example consider the dataset X=[1,...,17] and parameter T is set to 3.
The procedure can be visualized with the following table:
| | j=1,2,3,4,5 | j=6,7,8,9,10 | j=11,12,13,14,15 |
|-----|------------------|------------------|------------------|
| t=1 | validation | train | train |
| t=2 | train | validation | train |
| t=3 | train | train | validation |
"""
# Read the csv
df = pd.read_csv(filepath)
# Add a column of ones
df['constant'] = 1.0
# Don't Shuffle as this breaks the deterministic setup!!
# df = df.sample(frac=1, random_state=0).reset_index(drop=True)
# Total number of folds is
T = T_folds
# Extract features and label
labels = np.array(df.iloc[:, 0].values)
features = np.array(df.iloc[:, 1:].values)
# Validation Data
n_val = math.floor(len(labels)/T)
val_features = [features[t*n_val:(t + 1)*n_val] for t in range(T)]
val_labels = [labels[t*n_val:(t + 1)*n_val] for t in range(T)]
# Training data
n_train = n_val*(T - 1)
train_features = [np.concatenate(
[features[t_*n_val:(t_ + 1)*n_val] for t_ in range(T) if t_ != t]
) for t in range(T)]
train_labels = [np.concatenate(
[labels[t_*n_val:(t_ + 1)*n_val] for t_ in range(T) if t_ != t]
) for t in range(T)]
# Form into a dictionary
data = {
"path": filepath,
'd': features.shape[1],
"n_val": n_val,
"val_features": val_features,
"val_labels": val_labels,
"n_train": n_train,
"train_features": train_features,
"train_labels": train_labels,
}
return data
# //==================================================\\
# || Dimension ||
# \\==================================================//
def dimension(key='', data=None):
"""
If the argument 'key' is not specified, then:
- a dictionary mapping variable/function names (str) to the corresponding dimension (int) is returned.
If the first argument 'key' is specified, then:
- a single integer representing the dimension of the variable/function with the name {key} is returned.
"""
d = data['d']
n_train = data['n_train']
n = {
"x": 3 + 2*d,
"y": T_folds*(d + n_train),
"F": 1,
"G": 3 + d,
"H": 0,
"f": 1,
"g": T_folds*(3*n_train + 2*d),
"h": 0,
}
if key in n:
return n[key]
return n
# Feasible point
x0 = np.array([
942.69, # C
1.10, # Epsilon
394.61, # Lambda
0.00, -0.81, -0.04, -0.01, -0.38, 0.57, 1.29, -3.18, # w lower bound
0.02, -0.33, -0.01, -0.00, 0.09, 0.64, 1.37, -3.18 # w upper bound
])
x0 = np.array([
1.0, # C
1.0, # Epsilon
1.0, # Lambda
0.00615244, -0.8459975, -0.08478079, -0.00612273, -0.2105129, 0.50659832, 0.85530825, -0.49095779, # w lower bound
0.01608997, -0.08182366, -0.00441689, -0.00474096, -0.09918633, 0.65285064, 1.50846709, 0.15002266, # w upper bound
])