Commit: 372bf58aa1eba13e378314c8b9d343fd7fd4210b - myprojects-hg-reborn (hg)

Commit MetaInfo

• Revisão: 372bf58aa1eba13e378314c8b9d343fd7fd4210b (tree)
• Hora: 2013-06-25 02:04:38
• Autor: Lorenzo Isella <lorenzo.isella@gmai...>
• Commiter: Lorenzo Isella

Mensagem de Log

A code to carry out logistic regression in Python.
I still need to read and understand it and it may be possible that it can
be sped up/improved.

Mudança Sumário

• add: Python-codes/logistic_regression_updated.py

Diff

diff -r 90d01d56e457 -r 372bf58aa1eb Python-codes/logistic_regression_updated.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Python-codes/logistic_regression_updated.py	Mon Jun 24 19:04:38 2013 +0200
@@ -0,0 +1,164 @@
+__author__ = 'Miroslaw Horbal'
+__email__ = '[e-mail]'
+__date__ = '14-06-2013'
+
+from numpy import array, hstack
+from sklearn import metrics, cross_validation, linear_model
+from scipy import sparse
+from itertools import combinations
+
+import numpy as np
+import pandas as pd
+
+SEED = 25
+
+def group_data(data, degree=3, hash=hash):
+    """ 
+    numpy.array -> numpy.array
+    
+    Groups all columns of data into all combinations of triples
+    """
+    new_data = []
+    m,n = data.shape
+    for indicies in combinations(range(n), degree):
+        new_data.append([hash(tuple(v)) for v in data[:,indicies]])
+    return array(new_data).T
+
+def OneHotEncoder(data, keymap=None):
+     """
+     OneHotEncoder takes data matrix with categorical columns and
+     converts it to a sparse binary matrix.
+     
+     Returns sparse binary matrix and keymap mapping categories to indicies.
+     If a keymap is supplied on input it will be used instead of creating one
+     and any categories appearing in the data that are not in the keymap are
+     ignored
+     """
+     if keymap is None:
+          keymap = []
+          for col in data.T:
+               uniques = set(list(col))
+               keymap.append(dict((key, i) for i, key in enumerate(uniques)))
+     total_pts = data.shape[0]
+     outdat = []
+     for i, col in enumerate(data.T):
+          km = keymap[i]
+          num_labels = len(km)
+          spmat = sparse.lil_matrix((total_pts, num_labels))
+          for j, val in enumerate(col):
+               if val in km:
+                    spmat[j, km[val]] = 1
+          outdat.append(spmat)
+     outdat = sparse.hstack(outdat).tocsr()
+     return outdat, keymap
+
+def create_test_submission(filename, prediction):
+    content = ['id,ACTION']
+    for i, p in enumerate(prediction):
+        content.append('%i,%f' %(i+1,p))
+    f = open(filename, 'w')
+    f.write('\n'.join(content))
+    f.close()
+    print 'Saved'
+
+# This loop essentially from Paul's starter code
+def cv_loop(X, y, model, N):
+    mean_auc = 0.
+    for i in range(N):
+        X_train, X_cv, y_train, y_cv = cross_validation.train_test_split(
+                                       X, y, test_size=.20, 
+                                       random_state = i*SEED)
+        model.fit(X_train, y_train)
+        preds = model.predict_proba(X_cv)[:,1]
+        auc = metrics.auc_score(y_cv, preds)
+        print "AUC (fold %d/%d): %f" % (i + 1, N, auc)
+        mean_auc += auc
+    return mean_auc/N
+    
+def main(train='train.csv', test='test.csv', submit='logistic_pred.csv'):    
+    print "Reading dataset..."
+    train_data = pd.read_csv(train)
+    test_data = pd.read_csv(test)
+    all_data = np.vstack((train_data.ix[:,1:-1], test_data.ix[:,1:-1]))
+
+    num_train = np.shape(train_data)[0]
+    
+    # Transform data
+    print "Transforming data..."
+    dp = group_data(all_data, degree=2) 
+    dt = group_data(all_data, degree=3)
+
+    y = array(train_data.ACTION)
+    X = all_data[:num_train]
+    X_2 = dp[:num_train]
+    X_3 = dt[:num_train]
+
+    X_test = all_data[num_train:]
+    X_test_2 = dp[num_train:]
+    X_test_3 = dt[num_train:]
+
+    X_train_all = np.hstack((X, X_2, X_3))
+    X_test_all = np.hstack((X_test, X_test_2, X_test_3))
+    num_features = X_train_all.shape[1]
+    
+    model = linear_model.LogisticRegression()
+    
+    # Xts holds one hot encodings for each individual feature in memory
+    # speeding up feature selection 
+    Xts = [OneHotEncoder(X_train_all[:,[i]])[0] for i in range(num_features)]
+    
+    print "Performing greedy feature selection..."
+    score_hist = []
+    N = 10
+    good_features = set([])
+    # Greedy feature selection loop
+    while len(score_hist) < 2 or score_hist[-1][0] > score_hist[-2][0]:
+        scores = []
+        for f in range(len(Xts)):
+            if f not in good_features:
+                feats = list(good_features) + [f]
+                Xt = sparse.hstack([Xts[j] for j in feats]).tocsr()
+                score = cv_loop(Xt, y, model, N)
+                scores.append((score, f))
+                print "Feature: %i Mean AUC: %f" % (f, score)
+        good_features.add(sorted(scores)[-1][1])
+        score_hist.append(sorted(scores)[-1])
+        print "Current features: %s" % sorted(list(good_features))
+    
+    # Remove last added feature from good_features
+    good_features.remove(score_hist[-1][1])
+    good_features = sorted(list(good_features))
+    print "Selected features %s" % good_features
+    
+    print "Performing hyperparameter selection..."
+    # Hyperparameter selection loop
+    score_hist = []
+    Xt = sparse.hstack([Xts[j] for j in good_features]).tocsr()
+    Cvals = np.logspace(-4, 4, 15, base=2)
+    for C in Cvals:
+        model.C = C
+        score = cv_loop(Xt, y, model, N)
+        score_hist.append((score,C))
+        print "C: %f Mean AUC: %f" %(C, score)
+    bestC = sorted(score_hist)[-1][1]
+    print "Best C value: %f" % (bestC)
+    
+    print "Performing One Hot Encoding on entire dataset..."
+    Xt = np.vstack((X_train_all[:,good_features], X_test_all[:,good_features]))
+    Xt, keymap = OneHotEncoder(Xt)
+    X_train = Xt[:num_train]
+    X_test = Xt[num_train:]
+    
+    print "Training full model..."
+    model.fit(X_train, y)
+    
+    print "Making prediction and saving results..."
+    preds = model.predict_proba(X_test)[:,1]
+    create_test_submission(submit, preds)
+    
+if __name__ == "__main__":
+    args = { 'train':  'train.csv',
+             'test':   'test.csv',
+             'submit': 'logistic_regression_pred.csv' }
+    main(**args)
+