Debian - R: calculation distance matrix between two lists of strings

larry77
June 25, 2024
82 views
0 votes
2 Answers

Please consider the reprex at the end of the post.
I have two lists of dataframes. Each dataframe has a $keyword column, which is a vector of text.
I am looking for a computationally efficient way to get a distance matrix between the text in the keyword column of list_v1 and list_v2. The end result should be either a 5 by 10 or a 10 by 5 matrix or dataframe/tibble.
Any measure in the stringdist package (for instance the Levenshtein) would do for an example.
On top of that, I would like to calculate the jaccard index (see formula in the reprex) again between the text in the keyword column of list_v1 and list_v2.
Any suggestion is appreciated.

library(tidyverse)
library(stringdist)
#> 
#> Attaching package: 'stringdist'
#> The following object is masked from 'package:tidyr':
#> 
#>     extract


jaccard <- function(a, b) {
    intersection = length(intersect(a, b))
    union = length(a) + length(b) - intersection
    return (intersection/union)
}


list_v1 <- list(structure(list(keyword = c("lieutenant army", "General army", 
"air force", "government establishment", "commander", "army", 
"lieutenant", "force officer", "force", "officer", "Air", "border", 
"employment", "equivalent", "example", "group", "job", "leadership", 
"management", "member", "occupations", "person", "rank", "service", 
"task", "unit", "variety"), ngram = c(2L, 2L, 2L, 2L, 1L, 1L, 
1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
1L, 1L, 1L, 1L, 1L), freq = c(1L, 1L, 2L, 1L, 1L, 3L, 1L, 1L, 
4L, 2L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 2L, 2L, 1L, 1L, 1L, 
1L, 1L, 1L), rake = c(2.76190476190476, 2.42857142857143, 2.41666666666667, 
2, 1.5, 1.42857142857143, 1.33333333333333, 1.08333333333333, 
0.75, 0.333333333333333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0)), row.names = c(NA, -27L), class = "data.frame"), 
    structure(list(keyword = c("government establishment", "force occupation", 
    "force officer", "warrant officer", "force", "rank", "officer", 
    "activity", "army", "border", "discipline", "employment", 
    "example", "flight", "group", "job", "member", "occupations", 
    "person", "service", "task", "variety"), ngram = c(2L, 2L, 
    2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 2L, 2L, 1L, 1L, 1L, 1L
    ), rake = c(2, 2, 1.66666666666667, 1.66666666666667, 1, 
    1, 0.666666666666667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0)), row.names = c(NA, -22L), class = "data.frame"), 
    structure(list(keyword = c("government establishment", "air force", 
    "force example", "force occupation", "force", "Gunner", "army", 
    "border", "employment", "group", "job", "member", "navy", 
    "occupations", "officer", "person", "rank", "service", "task", 
    "variety"), ngram = c(2L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 2L, 1L, 1L, 
    1L, 1L, 2L, 1L), rake = c(2, 1.75, 1.75, 1.75, 0.75, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, 
    -20L), class = "data.frame"), structure(list(keyword = c("assemblie b", 
    "committee e", "government Secretary", "government class", 
    "government department", "community function", "repeal law", 
    "interest group", "parliament councils", "government", "community", 
    "councils", "law", "parliament", "policy", "interest", "Legislator", 
    "administrator", "agency", "agreement", "amend", "amending", 
    "board", "body", "concern", "constituency", "framework", 
    "implementation", "information", "interpretation", "legislator", 
    "matter", "meeting", "member", "occupations", "official", 
    "opinion", "order", "presiding", "proceeding", "regulation", 
    "representative", "rule", "service", "state", "statutory"
    ), ngram = c(2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 5L, 1L, 1L, 1L, 1L, 3L, 2L, 1L, 1L, 2L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 
    1L, 1L, 1L, 2L, 1L, 2L, 1L, 4L, 1L), rake = c(2, 2, 1.90909090909091, 
    1.90909090909091, 1.90909090909091, 1.5, 1.5, 1.33333333333333, 
    1, 0.909090909090909, 0.5, 0.5, 0.5, 0.5, 0.5, 0.333333333333333, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, -46L
    ), class = "data.frame"), structure(list(keyword = c("Police commissioner", 
    "government department", "policy matter", "government administration", 
    "government committee", "government e", "government policy", 
    "unit Group", "amendment class", "body tasnks", "document briefs", 
    "service commissioner", "commissioner inspector", "staff member", 
    "policy programme", "government official", "government manager", 
    "government legislation", "agency example", "government", 
    "budget", "control", "policy", "manager", "official", "agency", 
    "legislation", "Director", "accordance", "activity", "administrator", 
    "advising", "behalf", "commission", "conjunction", "consultation", 
    "country", "enterprise", "implementation", "interpretation", 
    "legislator", "objective", "occupations", "organization", 
    "preparation", "presentations", "procedure", "programm", 
    "regulation", "report", "state", "system", "task"), ngram = c(2L, 
    2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 
    2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 4L, 1L, 1L, 1L, 
    1L, 3L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 4L, 
    1L, 1L, 2L, 1L, 1L, 3L, 3L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 3L, 2L, 1L, 1L, 1L, 2L, 1L, 1L, 2L, 1L, 1L, 1L, 
    1L, 1L, 1L), rake = c(4, 3.02380952380952, 2.375, 2.19047619047619, 
    2.19047619047619, 2.19047619047619, 2.06547619047619, 2, 
    2, 2, 2, 2, 2, 2, 1.875, 1.69047619047619, 1.69047619047619, 
    1.44047619047619, 1.25, 1.19047619047619, 1, 1, 0.875, 0.5, 
    0.5, 0.25, 0.25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, -53L
    ), class = "data.frame"), structure(list(keyword = c("councils g", 
    "custom e", "regulation example", "surplus production", "government rule", 
    "village b", "village care", "village chiefs", "chiefs", 
    "rule", "village", "allocating", "authority", "community", 
    "connection", "dispute", "division", "duty", "head", "household", 
    "land", "member", "occasion", "occupations", "resource", 
    "responsibility", "rights", "task", "tradition", "use", "variety", 
    "violation"), ngram = c(2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 5L, 1L, 1L, 6L, 1L, 1L, 1L, 2L, 2L, 
    1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), rake = c(2, 
    2, 2, 2, 1.5, 1.375, 1.375, 0.875, 0.5, 0.5, 0.375, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, 
    -32L), class = "data.frame"), structure(list(keyword = c("interest group", 
    "of planning", "policy rule", "funding agency", "forum example", 
    "charity organization", "organization class", "employer organization", 
    "interest organization", "rights organization", "sport associations", 
    "board meeting", "party organization", "director", "interest", 
    "policy", "organization", "associations", "board", "party", 
    "act", "behalf", "body", "convention", "determining", "enterprise", 
    "functioning", "government", "hearing", "implementation", 
    "member", "membership", "negotiation", "objective", "occasion", 
    "occupations", "official", "operation", "performance", "procedure", 
    "regulation", "result", "section", "system"), ngram = c(2L, 
    2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 
    8L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L), rake = c(2.5, 
    2.5, 2.25, 2, 2, 1.72222222222222, 1.72222222222222, 1.72222222222222, 
    1.72222222222222, 1.72222222222222, 1.5, 1.5, 1.22222222222222, 
    1, 1, 0.75, 0.722222222222222, 0.5, 0.5, 0.5, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0)), row.names = c(NA, -44L), class = "data.frame"), structure(list(
        keyword = c("unit Group", "conventions seminar", "unit group", 
        "staff k", "subordinate staff", "body class", "government department", 
        "regulation example", "operation undertaken", "organization b", 
        "organization e", "interest organization", "manager note", 
        "board meeting", "objective", "sale manager", "body", 
        "example", "government", "operation", "policy", "sale", 
        "organization", "manager", "board", "Director", "activity", 
        "area", "budget", "consult", "coordinate", "director", 
        "directors", "enterprise", "enterprisis", "executive", 
        "expenditure", "forum", "function", "functioning", "guideline", 
        "hearing", "leadership", "legislation", "management", 
        "member", "negotiation", "occasion", "occupations", "performance", 
        "planning", "programme", "range", "recommendation", "report", 
        "resource", "responsibility", "result", "selection", 
        "support", "use"), ngram = c(2L, 2L, 2L, 2L, 2L, 2L, 
        2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 2L, 1L, 1L, 1L, 1L, 
        1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
        1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
        1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), 
        freq = c(3L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
        1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 3L, 1L, 7L, 5L, 3L, 
        3L, 2L, 1L, 1L, 1L, 1L, 4L, 1L, 6L, 2L, 2L, 1L, 1L, 1L, 
        1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 
        1L, 1L, 2L, 2L, 2L, 1L, 1L, 1L), rake = c(2, 2, 2, 2, 
        2, 1.5, 1.5, 1.5, 1.5, 1.3, 1.3, 1.3, 1.28571428571429, 
        1.25, 1, 0.785714285714286, 0.5, 0.5, 0.5, 0.5, 0.5, 
        0.5, 0.3, 0.285714285714286, 0.25, 0, 0, 0, 0, 0, 0, 
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, 
    -61L), class = "data.frame"), structure(list(keyword = c("management accountant", 
    "body example", "resource e", "staff h", "selection training", 
    "operation g", "Finance manager", "finance manager", "organization b", 
    "operation", "manager", "organization", "budget", "consultation", 
    "controller", "dealings", "departments", "enterprise", "enterprisis", 
    "expenditure", "occupations", "performance", "planning", 
    "procedure", "section", "service", "situation", "use"), ngram = c(2L, 
    2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 3L, 4L, 2L, 1L, 1L, 1L, 
    2L, 4L, 2L, 1L, 2L, 1L, 1L, 1L, 2L, 1L, 1L, 1L), rake = c(2, 
    2, 2, 2, 2, 1.75, 1.66666666666667, 1.66666666666667, 1.2, 
    0.75, 0.666666666666667, 0.2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, -28L), class = "data.frame"), 
    structure(list(keyword = c("resource manager", "manager", 
    "selection training", "relation", "activity e", "safety activity", 
    "body example", "organizing negotiation", "wage structure", 
    "safety health", "resource service", "procedure g", "health", 
    "management", "planning", "policy", "staff", "resource", 
    "safety", "organization", "procedure", "budget", "compliance", 
    "concern", "condition", "consult", "consultation", "dealings", 
    "departments", "determination", "development", "dismissal", 
    "employment", "enterprise", "enterprisis", "expenditure", 
    "implementation", "legislation", "level", "occupations", 
    "opportunity", "performance", "personnel", "practice", "programm", 
    "standard", "use", "worker"), ngram = c(2L, 1L, 2L, 1L, 2L, 
    2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 4L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 4L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 
    1L, 1L), rake = c(3.86666666666667, 3.2, 3, 2.33333333333333, 
    2.33333333333333, 2, 2, 2, 2, 1.66666666666667, 1.66666666666667, 
    1.33333333333333, 1, 1, 1, 1, 1, 0.666666666666667, 0.666666666666667, 
    0.4, 0.333333333333333, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, 
    -48L), class = "data.frame"))


list_v2 <- list(structure(list(keyword = c("user journey", "market data", 
"logistic firm", "management tool", "rights offence", "other shine", 
"usability testing", "user experience", "business requirement", 
"end journey", "health service", "house team", "stakeholder interest", 
"bank design", "design process", "constraint ability", "service design", 
"product designer", "product design", "experience", "business", 
"designer", "end", "interest", "service", "team", "design", "product", 
"ability", "action", "aesthetic", "agency", "app", "artefact", 
"attitude", "blueprint", "client", "consultant", "core", "curiosity", 
"define", "deliverable", "developer", "discipline", "eye", "flow", 
"future", "idea", "insight", "kind", "life", "lot", "mind", "mindset", 
"mockup", "money", "need", "opportunity", "output", "passion", 
"persona", "practice", "prototype", "reporting", "research", 
"role", "sense", "story", "strategy", "thing", "travel", "trend", 
"wireframe", "workshop"), ngram = c(2L, 2L, 2L, 2L, 2L, 2L, 2L, 
2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 1L, 1L, 1L, 
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 3L, 1L, 1L, 1L, 1L, 5L, 1L, 
5L, 4L, 5L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 3L, 1L, 1L, 1L, 1L, 2L, 
1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
1L, 2L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 2L, 1L, 2L, 1L, 1L
), rake = c(2.25, 2, 2, 2, 2, 2, 2, 1.91666666666667, 1.5, 1.5, 
1.5, 1.5, 1.5, 1.44444444444444, 1.44444444444444, 1.16666666666667, 
0.944444444444444, 0.833333333333333, 0.777777777777778, 0.666666666666667, 
0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.444444444444444, 0.333333333333333, 
0.166666666666667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, -74L), class = "data.frame"), 
    structure(list(keyword = c("employment Business", "employment agency", 
    "bank holiday", "construction environment", "pension contribution", 
    "minimising disruption", "home projects", "home scheme", 
    "property information", "market leader", "site operation", 
    "training opportunity", "roofing renewal", "communication skill", 
    "literacy skill", "principal contractor", "housing sector", 
    "job type", "site team", "ability", "opportunity", "client team", 
    "skill", "role", "contractor", "housing", "job", "team", 
    "client", "CV", "Cladding", "FTC", "access", "appointment", 
    "appreciation", "benefit", "candidate", "complaint", "customer", 
    "detail", "experience", "initiative", "issue", "leave", "level", 
    "location", "mixture", "offer", "party", "phone", "process", 
    "project", "recruitment", "refurbishment", "relation", "relationship", 
    "resident", "supply", "survey", "today", "touch", "units", 
    "vacancy", "value", "view", "worker", "year"), ngram = c(2L, 
    2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 
    2L, 2L, 2L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 2L, 1L, 1L, 1L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L), rake = c(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1.75, 
    1.75, 1.5, 1.5, 1.5, 1.5, 1, 1, 0.833333333333333, 0.75, 
    0.666666666666667, 0.5, 0.5, 0.5, 0.5, 0.333333333333333, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, 
    -67L), class = "data.frame"), structure(list(keyword = c("department standard", 
    "Package Benefits", "Wells area", "household name", "stock package", 
    "customer base", "business objective", "quality plant", "manager", 
    "product offer", "customer service", "retail business", "day", 
    "job", "business", "customer", "plant", "service", "role", 
    "retail", "product", "Assistants", "Tn2", "accessory", "advice", 
    "aspect", "belief", "candidate", "client", "company", "condition", 
    "confidence", "desire", "example", "experience", "garden", 
    "gardener", "gardening", "hand", "horticulturist", "individual", 
    "interview", "knowledge", "layout", "location", "love", "management", 
    "order", "passion", "people", "plenty", "position", "purveyor", 
    "recruit", "reputation", "responsibility", "sale", "sundry", 
    "support", "team", "thing", "training", "update", "willingness"
    ), ngram = c(2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 1L, 2L, 2L, 
    2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 4L, 
    2L, 3L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 3L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L
    ), rake = c(2.5, 2, 2, 2, 2, 1.66666666666667, 1.66666666666667, 
    1.5, 1.33333333333333, 1.25, 1.16666666666667, 1, 1, 1, 0.666666666666667, 
    0.666666666666667, 0.5, 0.5, 0.4, 0.333333333333333, 0.25, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0)), row.names = c(NA, -64L), class = "data.frame"), 
    structure(list(keyword = c("conduct work", "document responsibility", 
    "industry health", "Groundworkers", "duty", "experience", 
    "qualification", "reference", "site", "skills", "week"), 
        ngram = c(2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L), 
        freq = c(1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L), 
        rake = c(2, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0)), row.names = c(NA, 
    -11L), class = "data.frame"), structure(list(keyword = c("Pension scheme", 
    "transportation allowances", "job career", "employment condition", 
    "hour contract", "university degree", "holiday payment", 
    "hour week", "risk manager", "market risk", "Mrcp team", 
    "communication skill", "risk team", "team work", "trading activity", 
    "life balance", "vacation day", "work environment", "programme knowledge", 
    "portfolio self", "month salary", "it skill", "experience", 
    "risk", "market", "skill", "team", "management", "day", "environment", 
    "knowledge", "life", "portfolio", "trading", "it", "salary", 
    "13th", "Fluent", "ability", "addition", "ambition", "analysis", 
    "atmosphere", "attention", "attitude", "audit", "business", 
    "candidate", "capital", "challenge", "change", "choice", 
    "coach", "colleague", "competence", "concept", "coordinating", 
    "course", "demand", "departments", "detail", "development", 
    "difference", "econometric", "economics", "finance", "focus", 
    "gross", "guideline", "home", "idea", "impact", "initiative", 
    "instrument", "interest", "listen", "measure", "measurement", 
    "method", "mindset", "monitoring", "motivat", "other", "part", 
    "party", "passion", "performance", "personality", "policy", 
    "possibility", "professional", "profile", "project", "publication", 
    "regulation", "regulator", "regulatory", "responsibility", 
    "result", "role", "scope", "stakeholder", "starter", "subject", 
    "supervisor", "support", "technique", "thing", "think", "thinking", 
    "time", "understanding", "way"), ngram = c(2L, 2L, 2L, 2L, 
    2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 2L, 
    2L, 2L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 1L, 1L), freq = c(1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 
    1L, 3L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 
    1L, 3L, 3L, 3L, 2L, 1L, 2L, 1L, 1L, 1L, 1L, 2L, 2L, 1L, 1L, 
    1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 2L, 2L, 1L, 1L, 
    2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 3L, 1L, 1L, 
    1L, 1L, 1L, 1L, 1L, 1L, 3L, 1L, 1L, 1L, 1L, 1L, 2L, 1L, 1L, 
    1L, 1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 
    1L, 1L, 2L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 1L, 2L
    ), rake = c(2, 2, 2, 2, 2, 2, 2, 2, 2, 1.875, 1.71428571428571, 
    1.71428571428571, 1.71428571428571, 1.71428571428571, 1.5, 
    1.5, 1.5, 1.5, 1.5, 1.5, 1.33333333333333, 1.04761904761905, 
    1, 1, 0.875, 0.714285714285714, 0.714285714285714, 0.666666666666667, 
    0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.333333333333333, 0.333333333333333, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
    0)), row.names = c(NA, -113L), class = "data.frame"))


## see what they look like

list_v1[[5]] |> glimpse()
#> Rows: 53
#> Columns: 4
#> $ keyword <chr> "Police commissioner", "government department", "policy matter…
#> $ ngram   <int> 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1,…
#> $ freq    <int> 1, 4, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1,…
#> $ rake    <dbl> 4.000000, 3.023810, 2.375000, 2.190476, 2.190476, 2.190476, 2.…

list_v2[[5]] |> glimpse()
#> Rows: 113
#> Columns: 4
#> $ keyword <chr> "Pension scheme", "transportation allowances", "job career", "…
#> $ ngram   <int> 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,…
#> $ freq    <int> 1, 1, 1, 1, 2, 1, 1, 1, 1, 3, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1,…
#> $ rake    <dbl> 2.0000000, 2.0000000, 2.0000000, 2.0000000, 2.0000000, 2.00000…

sessionInfo()
#> R version 4.4.1 (2024-06-14)
#> Platform: x86_64-pc-linux-gnu
#> Running under: Debian GNU/Linux 12 (bookworm)
#> 
#> Matrix products: default
#> BLAS:   /usr/lib/x86_64-linux-gnu/blas/libblas.so.3.11.0 
#> LAPACK: /usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3.11.0
#> 
#> locale:
#>  [1] LC_CTYPE=en_GB.UTF-8       LC_NUMERIC=C              
#>  [3] LC_TIME=en_GB.UTF-8        LC_COLLATE=en_GB.UTF-8    
#>  [5] LC_MONETARY=en_GB.UTF-8    LC_MESSAGES=en_GB.UTF-8   
#>  [7] LC_PAPER=en_GB.UTF-8       LC_NAME=C                 
#>  [9] LC_ADDRESS=C               LC_TELEPHONE=C            
#> [11] LC_MEASUREMENT=en_GB.UTF-8 LC_IDENTIFICATION=C       
#> 
#> time zone: Europe/Brussels
#> tzcode source: system (glibc)
#> 
#> attached base packages:
#> [1] stats     graphics  grDevices utils     datasets  methods   base     
#> 
#> other attached packages:
#>  [1] stringdist_0.9.12 lubridate_1.9.3   forcats_1.0.0     stringr_1.5.1    
#>  [5] dplyr_1.1.4       purrr_1.0.2       readr_2.1.5       tidyr_1.3.1      
#>  [9] tibble_3.2.1      ggplot2_3.5.1     tidyverse_2.0.0  
#> 
#> loaded via a namespace (and not attached):
#>  [1] gtable_0.3.5      compiler_4.4.1    reprex_2.1.0      tidyselect_1.2.1 
#>  [5] parallel_4.4.1    scales_1.3.0      yaml_2.3.8        fastmap_1.1.1    
#>  [9] R6_2.5.1          generics_0.1.3    knitr_1.46        munsell_0.5.1    
#> [13] R.cache_0.16.0    tzdb_0.4.0        pillar_1.9.0      R.utils_2.12.3   
#> [17] rlang_1.1.3       utf8_1.2.4        stringi_1.8.4     xfun_0.43        
#> [21] fs_1.6.4          timechange_0.3.0  cli_3.6.2         withr_3.0.0      
#> [25] magrittr_2.0.3    digest_0.6.35     grid_4.4.1        hms_1.1.3        
#> [29] lifecycle_1.0.4   R.methodsS3_1.8.2 R.oo_1.26.0       vctrs_0.6.5      
#> [33] evaluate_0.23     glue_1.7.0        styler_1.10.3     fansi_1.0.6      
#> [37] colorspace_2.1-0  rmarkdown_2.26    tools_4.4.1       pkgconfig_2.0.3  
#> [41] htmltools_0.5.8.1

^{Created on 2024-06-25 with reprex v2.1.0}

Answers

- QianZhang
- June 25, 2024 at 2:47 pm
- 0 votes
0
Initialise as a matrix with dimensions first and then iterate through each pair of data frames in list_v1 and list_v2. For each pair, it extracts the keyword columns and calculates the Jaccard distance. Like:
```
# Initialize an empty matrix to store distances
n_v1 <- length(list_v1)
n_v2 <- length(list_v2)
dist_matrix <- matrix(NA, nrow = n_v1, ncol = n_v2)

# Loop through list_v1 and list_v2 to calculate distances
for (i in seq_along(list_v1)) {
  keywords_v1 <- list_v1[[i]]$keyword
  
  for (j in seq_along(list_v2)) {
    keywords_v2 <- list_v2[[j]]$keyword
    
    dist_matrix[i, j] <- calculate_jaccard_distance(keywords_v1, keywords_v2)
  }
}
```
Login or Signup to reply.

IIUC

sapply(list_v2, (y)sapply(list_v1, (x) jaccard(x$keyword, y$keyword)))
             [,1]        [,2]       [,3]       [,4]        [,5]
 [1,] 0.010000000 0.010752688 0.04597701 0.00000000 0.007194245
 [2,] 0.021276596 0.011363636 0.03614458 0.00000000 0.000000000
 [3,] 0.010752688 0.011627907 0.02439024 0.00000000 0.000000000
 [4,] 0.025641026 0.000000000 0.01851852 0.00000000 0.019230769
 [5,] 0.007936508 0.000000000 0.00862069 0.00000000 0.012195122
 [6,] 0.000000000 0.000000000 0.01052632 0.02380952 0.006944444
 [7,] 0.008547009 0.009090909 0.00000000 0.00000000 0.039735099
 [8,] 0.000000000 0.000000000 0.05042017 0.00000000 0.041916168
 [9,] 0.009900990 0.000000000 0.02222222 0.00000000 0.014388489
[10,] 0.016666667 0.036036036 0.02752294 0.00000000 0.032051282

Please signup or login to give your own answer.

Click here to cancel reply.

Debian – R: calculation distance matrix between two lists of strings

Answers