diff --git a/assignments/hwk02/Back_Project.m b/assignments/hwk02/Back_Project.m index e8e5147..6063014 100644 --- a/assignments/hwk02/Back_Project.m +++ b/assignments/hwk02/Back_Project.m @@ -4,17 +4,19 @@ function [] = Back_Project(training_data, test_data, n_components) % stack data - data = vertcat(training_data, test_data); + data = vertcat(training_data, test_data); % TODO: perform PCA + % for each number of principal components for n = 1:length(n_components) - % TODO: perform the back projection algorithm using the first n_components(n) principal components + % TODO: perform the back projection algorithm using the first n_components(n) principal components + - % TODO: plot first 5 images back projected using the first - % n_components(n) principal components + % TODO: plot first 5 images back projected using the first + % n_components(n) principal components end diff --git a/assignments/hwk02/Classify.m b/assignments/hwk02/Classify.m index d001b68..f510904 100644 --- a/assignments/hwk02/Classify.m +++ b/assignments/hwk02/Classify.m @@ -3,21 +3,29 @@ % these posterior probabilities are compared using the log odds. function [predictions] = Classify(data, m1, m2, S1, S2, pc1, pc2) - d = 8; - - % TODO: calculate P(x|C) * P(C) for both classes + % calculate P(x|C) * P(C) for both classes - pxC1 = exp(-1/2*(data-m1)./S1*(data-m1)') / (power(2*pi,d/2) * sqrt(det(S1))); - pxC2 = exp(-1/2*(data-m2)*(S2\(data-m2).')); + pxC1 = mvnpdf(data, m1, S1); + pxC2 = mvnpdf(data, m2, S2); - g1 = pxC1 * pc1; - g2 = pxC2 * pc2; + g1 = log(pxC1 * pc1); + g2 = log(pxC2 * pc2); % TODO: calculate log odds, if > 0 then data(i) belongs to class c1, else, c2 - for i = 1:length(data) - data(i) - end + log_odds = g1 - g2; + % for i = 1:length(data) + % if g1 > g2 + % predictions(i) = 1; + % else + % predictions(i) = 2; + % end + % end % TODO: get predictions from log odds calculation + [num_rows, ~] = size(data); + predictions = zeros(num_rows,1); + for i = 1:num_rows + predictions(i) = log_odds(i) > 0; + end end % Function end \ No newline at end of file diff --git a/assignments/hwk02/Eigenfaces.m b/assignments/hwk02/Eigenfaces.m index 36f30a1..a27fac1 100644 --- a/assignments/hwk02/Eigenfaces.m +++ b/assignments/hwk02/Eigenfaces.m @@ -2,12 +2,17 @@ % 5 eigenvectors function [] = Eigenfaces(training_data, test_data) - % stack data - data = vertcat(training_data, test_data); + % stack data + data = vertcat(training_data, test_data); - % TODO: perform PCA + % perform PCA + coeff = pca(data); - % TODO: show the first 5 eigenvectors (see homework for example) - imagesc(reshape(faces_data(i,1:end-1),32,30)') + % show the first 5 eigenvectors (see homework for example) + for i = 1:5 + subplot(3,2,i) + imagesc(reshape(coeff(:,i),32,30)'); + end + % pause; end % Function end \ No newline at end of file diff --git a/assignments/hwk02/Error_Rate.m b/assignments/hwk02/Error_Rate.m index 03c1670..62212fb 100644 --- a/assignments/hwk02/Error_Rate.m +++ b/assignments/hwk02/Error_Rate.m @@ -2,6 +2,16 @@ % predicted labels that are incorrrect. function [] = Error_Rate(predictions, labels) - % TODO: compute error rate and print it out + % compute error rate and print it out + c = 0; + [total_rows, ~] = size(predictions); + + for i = 1:total_rows + if predictions(i) == labels(i) + c = c + 1; + end + end + + fprintf('Rate: %.1f%% (%d / %d)\n', 100 * c / total_rows, c, total_rows); end % Function end \ No newline at end of file diff --git a/assignments/hwk02/KNN.m b/assignments/hwk02/KNN.m index 7939128..fbd7e01 100644 --- a/assignments/hwk02/KNN.m +++ b/assignments/hwk02/KNN.m @@ -6,18 +6,46 @@ function [test_err] = KNN(k, training_data, test_data, training_labels, test_lab n = length(test_data(:,1)); % get number of rows in test data preds = zeros(length(test_labels),1); % predict labels for each test point - % TODO: compute pairwise euclidean distance between the test data and the - % training data + % compute pairwise euclidean distance between the test data and the training data + pairwise_distance = pdist2(training_data, test_data); + + unique_classes = unique(training_labels); % for each data point (row) in the test data for t = 1:n - % TODO: compute k-nearest neighbors for data point + distances = pairwise_distance(:,t); + [~, smallest_indexes] = sort(distances, 'ascend'); + smallest_k_indexes = smallest_indexes(1:k); + distances_by_class = zeros(max(unique_classes), 2); + for i = 1:length(unique_classes) + class = unique_classes(i); + this_class_distances = distances(training_labels == class,:); + distances_by_class(i,1) = class; + distances_by_class(i,2) = mean(this_class_distances); + end + distances_by_class_table = array2table(distances_by_class); % TODO: classify test point using majority rule. Include tie-breaking % using whichever class is closer by distance. Fill in preds with the % predicted label. + smallest_k_labels = training_labels(smallest_k_indexes); + + labels_by_count = tabulate(smallest_k_labels); + labels_by_count_sorted = sortrows(labels_by_count, 2); + most_frequent_label = labels_by_count_sorted(1,:); + most_frequent_label_count = most_frequent_label(2); + labels_that_have_most_frequent_count = labels_by_count_sorted(labels_by_count_sorted(:,2) == most_frequent_label_count,1); + if length(labels_that_have_most_frequent_count) > 1 + common_indexes = find(ismember(distances_by_class, labels_that_have_most_frequent_count)); + common_distances = distances_by_class(common_indexes,:); + sorted_distances = sortrows(common_distances,2); + preds(t) = sorted_distances(1,1); + else + winning_label = mode(smallest_k_labels); + preds(t) = winning_label; + end end diff --git a/assignments/hwk02/KNN_Error.m b/assignments/hwk02/KNN_Error.m index 2bb86d7..bf8929b 100644 --- a/assignments/hwk02/KNN_Error.m +++ b/assignments/hwk02/KNN_Error.m @@ -2,13 +2,19 @@ % errors for k-nearest neighbors using different values of k. function [] = KNN_Error(neigenvectors, ks, training_data, test_data, training_labels, test_labels) - % TODO: perform PCA + % perform PCA + % TODO: project data using the number of eigenvectors defined by neigenvectors % TODO: compute test error for kNN with differents k's. Fill in % test_errors with the results for each k in ks. test_errors = zeros(1,length(ks)); + for i = 1:length(ks) + k = ks(i); + + test_errors(i) = KNN(k, training_data, test_data, training_labels, test_labels); + end % print error table fprintf("-----------------------------\n"); diff --git a/assignments/hwk02/Param_Est.m b/assignments/hwk02/Param_Est.m index 4ac24af..b6bd501 100644 --- a/assignments/hwk02/Param_Est.m +++ b/assignments/hwk02/Param_Est.m @@ -4,7 +4,7 @@ % S2: learned covariance matrix for features of class 2) function [m1, m2, S1, S2] = Param_Est(training_data, training_labels, part) - [num_rows, num_cols] = size(training_data); + [num_rows, ~] = size(training_data); class1_data = training_data(training_labels==1,:); class2_data = training_data(training_labels==2,:); @@ -14,11 +14,14 @@ function [m1, m2, S1, S2] = Param_Est(training_data, training_labels, part) S1 = cov(class1_data); S2 = cov(class2_data); - % Parameter estimation for 3 different models described in homework + % Model 3. + % Assume 𝑆1 and 𝑆2 are diagonal (the Naive Bayes model in equation (5.24)). if(strcmp(part, '3')) S1 = diag(diag(S1)); S2 = diag(diag(S2)); + % Model 2. + % Assume 𝑆1 = 𝑆2. In other words, shared S between two classes (the discriminant function is as equation (5.21) and (5.22) in the textbook). elseif(strcmp(part, '2')) P_C1 = length(class1_data) / num_rows; P_C2 = length(class2_data) / num_rows; @@ -27,6 +30,8 @@ function [m1, m2, S1, S2] = Param_Est(training_data, training_labels, part) S1 = S; S2 = S; + % Model 1. + % Assume independent 𝑆1 and 𝑆2 (the discriminant function is as equation (5.17) in the textbook). elseif(strcmp(part, '1')) end diff --git a/assignments/hwk02/Problem1.m b/assignments/hwk02/Problem1.m index a8c7f0d..b381a53 100644 --- a/assignments/hwk02/Problem1.m +++ b/assignments/hwk02/Problem1.m @@ -21,7 +21,7 @@ function [] = Problem1(training_file, test_file) fprintf('Model %s\n', part{i}); % Training for Multivariate Gaussian - [m1 m2 S1 S2] = Param_Est(training_data, training_labels, part(i)); + [m1, m2, S1, S2] = Param_Est(training_data, training_labels, part(i)); [predictions] = Classify(training_data, m1, m2, S1, S2, pc1, pc2); fprintf('training error\n'); Error_Rate(predictions, training_labels);