【linux + word2vec】计算两个词之间的相似度

桃扇骨 2022-06-13 05:26 199阅读 0赞

1、在linux上部署了word2vec之后，默认的是直接通过给出某一词的相似度较高的词的结果，但是没有直接给出计算两个词之间的相似度

2、通过修改distance.c文件的代码可以控制给出的词的数目，以及实现本文所提计算两个词之间的相似度的方法

//  Copyright 2013 Google Inc. All Rights Reserved.
    //
    //  Licensed under the Apache License, Version 2.0 (the "License");
    //  you may not use this file except in compliance with the License.
    //  You may obtain a copy of the License at
    //
    //      http://www.apache.org/licenses/LICENSE-2.0
    //
    //  Unless required by applicable law or agreed to in writing, software
    //  distributed under the License is distributed on an "AS IS" BASIS,
    //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    //  See the License for the specific language governing permissions and
    //  limitations under the License.
    
    #include <stdio.h>
    #include <string.h>
    #include <math.h>
    #include <malloc.h>
    
    const long long max_size = 2000;         // max length of strings
    const long long N = 10;                  // number of closest words that will be shown
    const long long max_w = 50;              // max length of vocabulary entries
    
    int main(int argc, char **argv) {
        FILE *f;
        char st1[max_size];
        char *bestw[N];
        char file_name[max_size], st[100][max_size];
        float len;
        long long words, size, a, b, c, cn, bi[100];
        float *M;
        char *vocab;
        if (argc < 2) {
            printf("Usage: ./distance <FILE>\nwhere FILE contains word projections in the BINARY FORMAT\n");
            return 0;
        }
        strcpy(file_name, argv[1]);
        f = fopen(file_name, "rb");
        if (f == NULL) {
            printf("Input file not found\n");
            return -1;
        }
        fscanf(f, "%lld", &words);
        fscanf(f, "%lld", &size);
        vocab = (char *)malloc((long long)words * max_w * sizeof(char));
        for (a = 0; a < N; a++) bestw[a] = (char *)malloc(max_size * sizeof(char));
        M = (float *)malloc((long long)words * (long long)size * sizeof(float));
        if (M == NULL) {
            printf("Cannot allocate memory: %lld MB    %lld  %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
            return -1;
        }
        for (b = 0; b < words; b++) {
            a = 0;
            while (1) {
                vocab[b * max_w + a] = fgetc(f);
                if (feof(f) || (vocab[b * max_w + a] == ' ')) break;
                if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
            }
            vocab[b * max_w + a] = 0;
            for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
            len = 0;
            for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
            len = sqrt(len);
            for (a = 0; a < size; a++) M[a + b * size] /= len;
        }
        fclose(f);
    
        while (1) {
            printf("Enter two word: ");
            a = 0;
            while (1) {
                st1[a] = fgetc(stdin);
                if ((st1[a] == '\n') || (a >= max_size - 1)) {
                    st1[a] = 0;
                    break;
                }
                a++;
            }
            if (!strcmp(st1, "EXIT")) break;
            cn = 0;
            b = 0;
            c = 0;
            while (1) {
                st[cn][b] = st1[c];
                b++;
                c++;
                st[cn][b] = 0;
                if (st1[c] == 0) break;
                if (st1[c] == ' ') {
                    cn++;
                    b = 0;
                    c++;
                }
            }
            cn++;
            if(cn != 2) {
                printf("please give two words\n");
                continue;
            }
            for (a = 0; a < cn; a++) {
                for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
                if (b == words) b = -1;
                bi[a] = b;
                printf("Word: %s  Position in vocabulary: %lld\n", st[a], bi[a]);
                if (b == -1) {
                    printf("Out of dictionary word!\n");
                    break;
                }
            }
            if (b == -1) continue;
            
            float first[max_size], second[max_size];
            for (a = 0; a < size; ++a) {
                first[a] = M[a + bi[0] * size];
                second[a] = M[a + bi[1] * size];
            }
            float product1 = 0.0, product2 = 0.0, sum12 = 0.0;
            for (a = 0; a < size; ++a) {
                product1 = product1 + first[a] * first[a];
                product2 = product2 + second[a] * second[a];
                sum12 = sum12 + first[a] * second[a];
            }
            float len1 = sqrt(product1);
            float len2 = sqrt(product2);
            float similarity = sum12/(len1 * len2);
            printf("the similarity between these two words is %f\n\n", similarity);
        }
        return 0;
    }

将上面的代码替换distance.c文件，然后在当前文件夹下打开命令输入命令make即可编译

如下图计算两个词之间的相似度：

![Center][]

[Center]: /images/20220613/36de22858a964f3ea11a3fbc51700965.png