training and synthetic data generation code

assets/len_to_p.csv

@@ -0,0 +1,251 @@
+len,p
+1,0.014734972701616804
+2,0.05048222747773489
+3,0.05624961536094529
+4,0.05972235654062228
+5,0.05244278768803355
+6,0.05518581363248727
+7,0.046578690556781516
+8,0.04875025276280738
+9,0.04442471185039959
+10,0.04181356215327536
+11,0.040160713186745564
+12,0.041162972666449804
+13,0.03785727473339019
+14,0.03527250028573187
+15,0.03326798132632338
+16,0.0307271656277749
+17,0.028151182929938547
+18,0.025794993977651372
+19,0.024731192249193356
+20,0.021856290057410126
+21,0.021135366572008825
+22,0.019113264112956403
+23,0.017073578154260045
+24,0.015992192926158093
+25,0.013952506967461734
+26,0.012572202245412905
+27,0.011288606771405713
+28,0.009758842302383443
+29,0.008993960067872309
+30,0.008176327334429372
+31,0.0072356101034788955
+32,0.006919107109888081
+33,0.005978389878937605
+34,0.004712377904574347
+35,0.00467721090528648
+36,0.004220039914544191
+37,0.003463949429855024
+38,0.003358448431991419
+39,0.003059528938044539
+40,0.00263752494659012
+41,0.0021891457056697995
+42,0.002364980702109141
+43,0.002013310709230458
+44,0.0019341849608327545
+45,0.0013099707234730928
+46,0.0013363459729389942
+47,0.001204469725609488
+48,0.0011341357270337517
+49,0.0008967584818406407
+50,0.000914341981484575
+51,0.000914341981484575
+52,0.0007736739843331018
+53,0.0006505894868255629
+54,0.0006681729864694971
+55,0.0005011297398521228
+56,0.0005714637384278593
+57,0.00044837924092032037
+58,0.000395628741988518
+59,0.00031650299359081436
+60,0.00031650299359081436
+61,0.0002813359943029461
+62,0.00026375249465901196
+63,0.0002725442444809791
+64,0.00020221024590524252
+65,0.00032529474341278143
+66,0.00023737724519311078
+67,0.00023737724519311078
+68,0.00022858549537114374
+69,0.00020221024590524252
+70,0.00012308449750753894
+71,0.00010550099786360479
+72,8.791749821967066e-05
+73,0.00012308449750753894
+74,0.00011429274768557187
+75,7.912574839770359e-05
+76,3.516699928786826e-05
+77,7.033399857573652e-05
+78,8.791749821967066e-05
+79,3.516699928786826e-05
+80,2.6375249465901198e-05
+81,6.154224875376947e-05
+82,0.00011429274768557187
+83,7.033399857573652e-05
+84,5.2750498931802396e-05
+85,4.395874910983533e-05
+86,3.516699928786826e-05
+87,8.791749821967066e-05
+88,6.154224875376947e-05
+89,1.758349964393413e-05
+90,1.758349964393413e-05
+91,1.758349964393413e-05
+92,8.791749821967065e-06
+93,3.516699928786826e-05
+94,2.6375249465901198e-05
+95,2.6375249465901198e-05
+96,1.758349964393413e-05
+97,1.758349964393413e-05
+98,4.395874910983533e-05
+99,4.395874910983533e-05
+100,8.791749821967065e-06
+101,8.791749821967065e-06
+102,2.6375249465901198e-05
+103,2.6375249465901198e-05
+104,8.791749821967065e-06
+105,8.791749821967065e-06
+106,1.758349964393413e-05
+107,1.758349964393413e-05
+108,8.791749821967065e-06
+109,8.791749821967065e-06
+110,8.791749821967065e-06
+111,8.791749821967065e-06
+112,8.791749821967065e-06
+113,8.791749821967065e-06
+114,3.516699928786826e-05
+115,2.6375249465901198e-05
+116,2.6375249465901198e-05
+117,2.6375249465901198e-05
+118,8.791749821967065e-06
+119,8.791749821967065e-06
+120,8.791749821967065e-06
+121,8.791749821967065e-06
+122,1.758349964393413e-05
+123,8.791749821967065e-06
+124,8.791749821967065e-06
+125,8.791749821967065e-06
+126,1.758349964393413e-05
+127,1.758349964393413e-05
+128,1.758349964393413e-05
+129,1.758349964393413e-05
+130,1.758349964393413e-05
+131,8.791749821967065e-06
+132,1.758349964393413e-05
+133,8.791749821967065e-06
+134,8.791749821967065e-06
+135,8.791749821967065e-06
+136,8.791749821967065e-06
+137,8.791749821967065e-06
+138,8.791749821967065e-06
+139,8.791749821967065e-06
+140,8.791749821967065e-06
+141,8.791749821967065e-06
+142,8.791749821967065e-06
+143,8.791749821967065e-06
+144,8.791749821967065e-06
+145,8.791749821967065e-06
+146,8.791749821967065e-06
+147,8.791749821967065e-06
+148,8.791749821967065e-06
+149,8.791749821967065e-06
+150,8.791749821967065e-06
+151,8.791749821967065e-06
+152,8.791749821967065e-06
+153,8.791749821967065e-06
+154,8.791749821967065e-06
+155,8.791749821967065e-06
+156,8.791749821967065e-06
+157,8.791749821967065e-06
+158,8.791749821967065e-06
+159,8.791749821967065e-06
+160,8.791749821967065e-06
+161,8.791749821967065e-06
+162,8.791749821967065e-06
+163,8.791749821967065e-06
+164,8.791749821967065e-06
+165,8.791749821967065e-06
+166,8.791749821967065e-06
+167,8.791749821967065e-06
+168,8.791749821967065e-06
+169,8.791749821967065e-06
+170,8.791749821967065e-06
+171,8.791749821967065e-06
+172,8.791749821967065e-06
+173,8.791749821967065e-06
+174,8.791749821967065e-06
+175,8.791749821967065e-06
+176,8.791749821967065e-06
+177,8.791749821967065e-06
+178,8.791749821967065e-06
+179,8.791749821967065e-06
+180,8.791749821967065e-06
+181,8.791749821967065e-06
+182,8.791749821967065e-06
+183,8.791749821967065e-06
+184,8.791749821967065e-06
+185,8.791749821967065e-06
+186,8.791749821967065e-06
+187,8.791749821967065e-06
+188,8.791749821967065e-06
+189,8.791749821967065e-06
+190,8.791749821967065e-06
+191,8.791749821967065e-06
+192,8.791749821967065e-06
+193,8.791749821967065e-06
+194,8.791749821967065e-06
+195,8.791749821967065e-06
+196,8.791749821967065e-06
+197,8.791749821967065e-06
+198,8.791749821967065e-06
+199,8.791749821967065e-06
+200,8.791749821967065e-06
+201,8.791749821967065e-06
+202,8.791749821967065e-06
+203,8.791749821967065e-06
+204,8.791749821967065e-06
+205,8.791749821967065e-06
+206,8.791749821967065e-06
+207,8.791749821967065e-06
+208,8.791749821967065e-06
+209,8.791749821967065e-06
+210,8.791749821967065e-06
+211,8.791749821967065e-06
+212,8.791749821967065e-06
+213,8.791749821967065e-06
+214,8.791749821967065e-06
+215,8.791749821967065e-06
+216,8.791749821967065e-06
+217,8.791749821967065e-06
+218,8.791749821967065e-06
+219,8.791749821967065e-06
+220,8.791749821967065e-06
+221,8.791749821967065e-06
+222,8.791749821967065e-06
+223,8.791749821967065e-06
+224,8.791749821967065e-06
+225,8.791749821967065e-06
+226,8.791749821967065e-06
+227,8.791749821967065e-06
+228,8.791749821967065e-06
+229,8.791749821967065e-06
+230,8.791749821967065e-06
+231,8.791749821967065e-06
+232,8.791749821967065e-06
+233,8.791749821967065e-06
+234,8.791749821967065e-06
+235,8.791749821967065e-06
+236,8.791749821967065e-06
+237,8.791749821967065e-06
+238,8.791749821967065e-06
+239,8.791749821967065e-06
+240,8.791749821967065e-06
+241,8.791749821967065e-06
+242,8.791749821967065e-06
+243,8.791749821967065e-06
+244,8.791749821967065e-06
+245,8.791749821967065e-06
+246,8.791749821967065e-06
+247,8.791749821967065e-06
+248,8.791749821967065e-06
+249,8.791749821967065e-06
+250,8.791749821967065e-06

assets/lines_example.csv

@@ -0,0 +1,6 @@
+source,id,line
+cc-100,cc-100_446088,発展を遂げた貨幣経済に対して、後戻りする形の改革が、民衆に受け入れられるはずもありません。
+cc-100,cc-100_446387,東京都渋谷区本町１丁目４−１４ ホームヘルパー（パート：茂原）
+cc-100,cc-100_446430,同時に、発表しあう場を増やしたいです。まず、自分の考えを発表するためには、しっかりと自分の考えを持っていなくてはいけません。そのために、ますますノートの必要性を感じることでしょう。また、質問や意見に答えることで、考えが深まります。友達の意見を聞くことが、より理解を深めることを実感してほしいです。
+cc-100,cc-100_446493,※特典の数に限りがございますので、対象商品はお早めにお買い求めください。特典は無くなり次第終了となります。
+cc-100,cc-100_446543,ハリウッドスターってもっと豪華な生活を送っているのかと思えば、キアヌ・リーブスってかなり質素なんですね。