forked from piskvorky/gensim
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtut2.html
More file actions
426 lines (395 loc) · 31.6 KB
/
Copy pathtut2.html
File metadata and controls
426 lines (395 loc) · 31.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Topics and Transformations — gensim</title>
<link rel="stylesheet" href="_static/default.css" type="text/css" />
<link rel="stylesheet" href="_static/pygments.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '',
VERSION: '0.8.6',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script>
<script type="text/javascript" src="_static/jquery.js"></script>
<script type="text/javascript" src="_static/underscore.js"></script>
<script type="text/javascript" src="_static/doctools.js"></script>
<link rel="author" title="About these documents" href="about.html" />
<link rel="top" title="gensim" href="index.html" />
<link rel="up" title="Tutorials" href="tutorial.html" />
<link rel="next" title="Similarity Queries" href="tut3.html" />
<link rel="prev" title="Corpora and Vector Spaces" href="tut1.html" />
<!-- twitter search widget
<script type="text/javascript" src="_static/widget.js"></script>
-->
<meta property="og:title" content="#gensim" />
<meta property="og:description" content="Efficient topic modelling in Python" />
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-24066335-1']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
</head>
<body>
<div class="related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="genindex.html" title="General Index"
accesskey="I">index</a></li>
<li class="right" >
<a href="py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="right" >
<a href="tut3.html" title="Similarity Queries"
accesskey="N">next</a> |</li>
<li class="right" >
<a href="tut1.html" title="Corpora and Vector Spaces"
accesskey="P">previous</a> |</li>
<li><a href="index.html">Gensim home</a>| </li>
<li><a href="tutorial.html">Tutorials</a>| </li>
<li><a href="http://groups.google.com/group/gensim">Support</a>| </li>
<li><a href="https://github.com/piskvorky/gensim/wiki">Contribute</a>| </li>
<li><a href="apiref.html">API reference</a>»</li>
<li><a href="tutorial.html" accesskey="U">Tutorials</a> »</li>
</ul>
</div>
<div class="sphinxsidebar">
<div class="sphinxsidebarwrapper">
<h3><a href="index.html">Table Of Contents</a></h3>
<ul>
<li><a class="reference internal" href="#">Topics and Transformations</a><ul>
<li><a class="reference internal" href="#transformation-interface">Transformation interface</a><ul>
<li><a class="reference internal" href="#creating-a-transformation">Creating a transformation</a></li>
<li><a class="reference internal" href="#transforming-vectors">Transforming vectors</a></li>
</ul>
</li>
<li><a class="reference internal" href="#available-transformations">Available transformations</a></li>
</ul>
</li>
</ul>
<h4>Previous topic</h4>
<p class="topless"><a href="tut1.html"
title="previous chapter">Corpora and Vector Spaces</a></p>
<h4>Next topic</h4>
<p class="topless"><a href="tut3.html"
title="next chapter">Similarity Queries</a></p>
<div id="searchbox" style="display: none">
<h3>Quick search</h3>
<form class="search" action="search.html" method="get">
<input type="text" name="q" size="24" />
<input type="submit" value="Go" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
<p class="searchtip" style="font-size: 90%">
Enter search terms or a module, class or function name.
</p>
</div>
<script type="text/javascript">$('#searchbox').show(0);</script>
</div>
</div>
<div class="document">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body">
<div class="section" id="topics-and-transformations">
<span id="tut2"></span><h1>Topics and Transformations<a class="headerlink" href="#topics-and-transformations" title="Permalink to this headline">¶</a></h1>
<p>Don’t forget to set</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="kn">import</span> <span class="nn">logging</span>
<span class="gp">>>> </span><span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">(</span><span class="n">format</span><span class="o">=</span><span class="s">'</span><span class="si">%(asctime)s</span><span class="s"> : </span><span class="si">%(levelname)s</span><span class="s"> : </span><span class="si">%(message)s</span><span class="s">'</span><span class="p">,</span> <span class="n">level</span><span class="o">=</span><span class="n">logging</span><span class="o">.</span><span class="n">INFO</span><span class="p">)</span>
</pre></div>
</div>
<p>if you want to see logging events.</p>
<div class="section" id="transformation-interface">
<h2>Transformation interface<a class="headerlink" href="#transformation-interface" title="Permalink to this headline">¶</a></h2>
<p>In the previous tutorial on <a class="reference internal" href="tut1.html"><em>Corpora and Vector Spaces</em></a>, we created a corpus of documents represented
as a stream of vectors. To continue, let’s fire up gensim and use that corpus:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">gensim</span> <span class="kn">import</span> <span class="n">corpora</span><span class="p">,</span> <span class="n">models</span><span class="p">,</span> <span class="n">similarities</span>
<span class="gp">>>> </span><span class="n">dictionary</span> <span class="o">=</span> <span class="n">corpora</span><span class="o">.</span><span class="n">Dictionary</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s">'/tmp/deerwester.dict'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">corpus</span> <span class="o">=</span> <span class="n">corpora</span><span class="o">.</span><span class="n">MmCorpus</span><span class="p">(</span><span class="s">'/tmp/deerwester.mm'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="k">print</span> <span class="n">corpus</span>
<span class="go">MmCorpus(9 documents, 12 features, 28 non-zero entries)</span>
</pre></div>
</div>
<p>In this tutorial, I will show how to transform documents from one vector representation
into another. This process serves two goals:</p>
<ol class="arabic simple">
<li>To bring out hidden structure in the corpus, discover relationships between
words and use them to describe the documents in a new and
(hopefully) more semantic way.</li>
<li>To make the document representation more compact. This both improves efficiency
(new representation consumes less resources) and efficacy (marginal data
trends are ignored, noise-reduction).</li>
</ol>
<div class="section" id="creating-a-transformation">
<h3>Creating a transformation<a class="headerlink" href="#creating-a-transformation" title="Permalink to this headline">¶</a></h3>
<p>The transformations are standard Python objects, typically initialized by means of
a <em class="dfn">training corpus</em>:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">tfidf</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">TfidfModel</span><span class="p">(</span><span class="n">corpus</span><span class="p">)</span> <span class="c"># step 1 -- initialize a model</span>
</pre></div>
</div>
<p>We used our old corpus from tutorial 1 to initialize (train) the transformation model. Different
transformations may require different initialization parameters; in case of TfIdf, the
“training” consists simply of going through the supplied corpus once and computing document frequencies
of all its features. Training other models, such as Latent Semantic Analysis or Latent Dirichlet
Allocation, is much more involved and, consequently, takes much more time.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Transformations always convert between two specific vector
spaces. The same vector space (= the same set of feature ids) must be used for training
as well as for subsequent vector transformations. Failure to use the same input
feature space, such as applying a different string preprocessing, using different
feature ids, or using bag-of-words input vectors where TfIdf vectors are expected, will
result in feature mismatch during transformation calls and consequently in either
garbage output and/or runtime exceptions.</p>
</div>
</div>
<div class="section" id="transforming-vectors">
<h3>Transforming vectors<a class="headerlink" href="#transforming-vectors" title="Permalink to this headline">¶</a></h3>
<p>From now on, <tt class="docutils literal"><span class="pre">tfidf</span></tt> is treated as a read-only object that can be used to convert
any vector from the old representation (bag-of-words integer counts) to the new representation
(TfIdf real-valued weights):</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">doc_bow</span> <span class="o">=</span> <span class="p">[(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">),</span> <span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)]</span>
<span class="gp">>>> </span><span class="k">print</span> <span class="n">tfidf</span><span class="p">[</span><span class="n">doc_bow</span><span class="p">]</span> <span class="c"># step 2 -- use the model to transform vectors</span>
<span class="go">[(0, 0.70710678), (1, 0.70710678)]</span>
</pre></div>
</div>
<p>Or to apply a transformation to a whole corpus:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">corpus_tfidf</span> <span class="o">=</span> <span class="n">tfidf</span><span class="p">[</span><span class="n">corpus</span><span class="p">]</span>
<span class="gp">>>> </span><span class="k">for</span> <span class="n">doc</span> <span class="ow">in</span> <span class="n">corpus_tfidf</span><span class="p">:</span>
<span class="gp">>>> </span> <span class="k">print</span> <span class="n">doc</span>
<span class="go">[(0, 0.57735026918962573), (1, 0.57735026918962573), (2, 0.57735026918962573)]</span>
<span class="go">[(0, 0.44424552527467476), (3, 0.44424552527467476), (4, 0.44424552527467476), (5, 0.32448702061385548), (6, 0.44424552527467476), (7, 0.32448702061385548)]</span>
<span class="go">[(2, 0.5710059809418182), (5, 0.41707573620227772), (7, 0.41707573620227772), (8, 0.5710059809418182)]</span>
<span class="go">[(1, 0.49182558987264147), (5, 0.71848116070837686), (8, 0.49182558987264147)]</span>
<span class="go">[(3, 0.62825804686700459), (6, 0.62825804686700459), (7, 0.45889394536615247)]</span>
<span class="go">[(9, 1.0)]</span>
<span class="go">[(9, 0.70710678118654746), (10, 0.70710678118654746)]</span>
<span class="go">[(9, 0.50804290089167492), (10, 0.50804290089167492), (11, 0.69554641952003704)]</span>
<span class="go">[(4, 0.62825804686700459), (10, 0.45889394536615247), (11, 0.62825804686700459)]</span>
</pre></div>
</div>
<p>In this particular case, we are transforming the same corpus that we used
for training, but this is only incidental. Once the transformation model has been initialized,
it can be used on any vectors (provided they come from the same vector space, of course),
even if they were not used in the training corpus at all. This is achieved by a process called
folding-in for LSA, by topic inference for LDA etc.</p>
<div class="admonition note">
<p class="first admonition-title">Note</p>
<p class="last">Calling <tt class="docutils literal"><span class="pre">model[corpus]</span></tt> only creates a wrapper around the old <tt class="docutils literal"><span class="pre">corpus</span></tt>
document stream – actual conversions are done on-the-fly, during document iteration.
We cannot convert the entire corpus at the time of calling <tt class="docutils literal"><span class="pre">corpus_transformed</span> <span class="pre">=</span> <span class="pre">model[corpus]</span></tt>,
because that would mean storing the result in main memory, and that contradicts gensim’s objective of memory-indepedence.
If you will be iterating over the transformed <tt class="docutils literal"><span class="pre">corpus_transformed</span></tt> multiple times, and the
transformation is costly, <a class="reference internal" href="tut1.html#corpus-formats"><em>serialize the resulting corpus to disk first</em></a> and continue
using that.</p>
</div>
<p>Transformations can also be serialized, one on top of another, in a sort of chain:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">lsi</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">LsiModel</span><span class="p">(</span><span class="n">corpus_tfidf</span><span class="p">,</span> <span class="n">id2word</span><span class="o">=</span><span class="n">dictionary</span><span class="p">,</span> <span class="n">num_topics</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span> <span class="c"># initialize an LSI transformation</span>
<span class="gp">>>> </span><span class="n">corpus_lsi</span> <span class="o">=</span> <span class="n">lsi</span><span class="p">[</span><span class="n">corpus_tfidf</span><span class="p">]</span> <span class="c"># create a double wrapper over the original corpus: bow->tfidf->fold-in-lsi</span>
</pre></div>
</div>
<p>Here we transformed our Tf-Idf corpus via <a class="reference external" href="http://en.wikipedia.org/wiki/Latent_semantic_indexing">Latent Semantic Indexing</a>
into a latent 2-D space (2-D because we set <tt class="docutils literal"><span class="pre">num_topics=2</span></tt>). Now you’re probably wondering: what do these two latent
dimensions stand for? Let’s inspect with <tt class="xref py py-func docutils literal"><span class="pre">models.LsiModel.print_topics()</span></tt>:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">lsi</span><span class="o">.</span><span class="n">print_topics</span><span class="p">(</span><span class="mi">2</span><span class="p">)</span>
<span class="go">topic #0(1.594): -0.703*"trees" + -0.538*"graph" + -0.402*"minors" + -0.187*"survey" + -0.061*"system" + -0.060*"response" + -0.060*"time" + -0.058*"user" + -0.049*"computer" + -0.035*"interface"</span>
<span class="go">topic #1(1.476): -0.460*"system" + -0.373*"user" + -0.332*"eps" + -0.328*"interface" + -0.320*"response" + -0.320*"time" + -0.293*"computer" + -0.280*"human" + -0.171*"survey" + 0.161*"trees"</span>
</pre></div>
</div>
<p>(the topics are printed to log – see the note at the top of this page about activating
logging)</p>
<p>It appears that according to LSI, “trees”, “graph” and “minors” are all related
words (and contribute the most to the direction of the first topic), while the
second topic practically concerns itself with all the other words. As expected,
the first five documents are more strongly related to the second topic while the
remaining four documents to the first topic:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="k">for</span> <span class="n">doc</span> <span class="ow">in</span> <span class="n">corpus_lsi</span><span class="p">:</span> <span class="c"># both bow->tfidf and tfidf->lsi transformations are actually executed here, on the fly</span>
<span class="gp">>>> </span> <span class="k">print</span> <span class="n">doc</span>
<span class="go">[(0, -0.066), (1, 0.520)] # "Human machine interface for lab abc computer applications"</span>
<span class="go">[(0, -0.197), (1, 0.761)] # "A survey of user opinion of computer system response time"</span>
<span class="go">[(0, -0.090), (1, 0.724)] # "The EPS user interface management system"</span>
<span class="go">[(0, -0.076), (1, 0.632)] # "System and human system engineering testing of EPS"</span>
<span class="go">[(0, -0.102), (1, 0.574)] # "Relation of user perceived response time to error measurement"</span>
<span class="go">[(0, -0.703), (1, -0.161)] # "The generation of random binary unordered trees"</span>
<span class="go">[(0, -0.877), (1, -0.168)] # "The intersection graph of paths in trees"</span>
<span class="go">[(0, -0.910), (1, -0.141)] # "Graph minors IV Widths of trees and well quasi ordering"</span>
<span class="go">[(0, -0.617), (1, 0.054)] # "Graph minors A survey"</span>
</pre></div>
</div>
<p>Model persistency is achieved with the <tt class="xref py py-func docutils literal"><span class="pre">save()</span></tt> and <tt class="xref py py-func docutils literal"><span class="pre">load()</span></tt> functions:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">lsi</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="s">'/tmp/model.lsi'</span><span class="p">)</span> <span class="c"># same for tfidf, lda, ...</span>
<span class="gp">>>> </span><span class="n">lsi</span> <span class="o">=</span> <span class="n">models</span><span class="o">.</span><span class="n">LsiModel</span><span class="o">.</span><span class="n">load</span><span class="p">(</span><span class="s">'/tmp/model.lsi'</span><span class="p">)</span>
</pre></div>
</div>
<p>The next question might be: just how exactly similar are those documents to each other?
Is there a way to formalize the similarity, so that for a given input document, we can
order some other set of documents according to their similarity? Similarity queries
are covered in the <a class="reference internal" href="tut3.html"><em>next tutorial</em></a>.</p>
</div>
</div>
<div class="section" id="available-transformations">
<span id="transformations"></span><h2>Available transformations<a class="headerlink" href="#available-transformations" title="Permalink to this headline">¶</a></h2>
<p>Gensim implements several popular Vector Space Model algorithms:</p>
<ul>
<li><p class="first"><a class="reference external" href="http://en.wikipedia.org/wiki/Tf%E2%80%93idf">Term Frequency * Inverse Document Frequency, Tf-Idf</a>
expects a bag-of-words (integer values) training corpus during initialization.
During transformation, it will take a vector and return another vector of the
same dimensionality, except that features which were rare in the training corpus
will have their value increased.
It therefore converts integer-valued vectors into real-valued ones, while leaving
the number of dimensions intact. It can also optionally normalize the resulting
vectors to (Euclidean) unit length.</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">tfidfmodel</span><span class="o">.</span><span class="n">TfidfModel</span><span class="p">(</span><span class="n">bow_corpus</span><span class="p">,</span> <span class="n">normalize</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
</pre></div>
</div>
</li>
<li><p class="first"><a class="reference external" href="http://en.wikipedia.org/wiki/Latent_semantic_indexing">Latent Semantic Indexing, LSI (or sometimes LSA)</a>
transforms documents from either bag-of-words or (preferrably) TfIdf-weighted space into
a latent space of a lower dimensionality. For the toy corpus above we used only
2 latent dimensions, but on real corpora, target dimensionality of 200–500 is recommended
as a “golden standard” <a class="footnote-reference" href="#id6" id="id1">[1]</a>.</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">lsimodel</span><span class="o">.</span><span class="n">LsiModel</span><span class="p">(</span><span class="n">tfidf_corpus</span><span class="p">,</span> <span class="n">id2word</span><span class="o">=</span><span class="n">dictionary</span><span class="p">,</span> <span class="n">num_topics</span><span class="o">=</span><span class="mi">300</span><span class="p">)</span>
</pre></div>
</div>
<p>LSI training is unique in that we can continue “training” at any point, simply
by providing more training documents. This is done by incremental updates to
the underlying model, in a process called <cite>online training</cite>. Because of this feature, the
input document stream may even be infinite – just keep feeding LSI new documents
as they arrive, while using the computed transformation model as read-only in the meanwhile!</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">add_documents</span><span class="p">(</span><span class="n">another_tfidf_corpus</span><span class="p">)</span> <span class="c"># now LSI has been trained on tfidf_corpus + another_tfidf_corpus</span>
<span class="gp">>>> </span><span class="n">lsi_vec</span> <span class="o">=</span> <span class="n">model</span><span class="p">[</span><span class="n">tfidf_vec</span><span class="p">]</span> <span class="c"># convert some new document into the LSI space, without affecting the model</span>
<span class="gp">>>> </span><span class="o">...</span>
<span class="gp">>>> </span><span class="n">model</span><span class="o">.</span><span class="n">add_documents</span><span class="p">(</span><span class="n">more_documents</span><span class="p">)</span> <span class="c"># tfidf_corpus + another_tfidf_corpus + more_documents</span>
<span class="gp">>>> </span><span class="n">lsi_vec</span> <span class="o">=</span> <span class="n">model</span><span class="p">[</span><span class="n">tfidf_vec</span><span class="p">]</span>
<span class="gp">>>> </span><span class="o">...</span>
</pre></div>
</div>
<p>See the <a class="reference internal" href="models/lsimodel.html#module-gensim.models.lsimodel" title="gensim.models.lsimodel: Latent Semantic Indexing"><tt class="xref py py-mod docutils literal"><span class="pre">gensim.models.lsimodel</span></tt></a> documentation for details on how to make
LSI gradually “forget” old observations in infinite streams. If you want to get dirty,
there are also parameters you can tweak that affect speed vs. memory footprint vs. numerical
precision of the LSI algorithm.</p>
<p><cite>gensim</cite> uses a novel online incremental streamed distributed training algorithm (quite a mouthful!),
which I published in <a class="footnote-reference" href="#id10" id="id2">[5]</a>. <cite>gensim</cite> also executes a stochastic multi-pass algorithm
from Halko et al. <a class="footnote-reference" href="#id9" id="id3">[4]</a> internally, to accelerate in-core part
of the computations.
See also <a class="reference internal" href="wiki.html"><em>Experiments on the English Wikipedia</em></a> for further speed-ups by distributing the computation across
a cluster of computers.</p>
</li>
<li><p class="first"><a class="reference external" href="http://www.cis.hut.fi/ella/publications/randproj_kdd.pdf">Random Projections, RP</a> aim to
reduce vector space dimensionality. This is a very efficient (both memory- and
CPU-friendly) approach to approximating TfIdf distances between documents, by throwing in a little randomness.
Recommended target dimensionality is again in the hundreds/thousands, depending on your dataset.</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">rpmodel</span><span class="o">.</span><span class="n">RpModel</span><span class="p">(</span><span class="n">tfidf_corpus</span><span class="p">,</span> <span class="n">num_topics</span><span class="o">=</span><span class="mi">500</span><span class="p">)</span>
</pre></div>
</div>
</li>
<li><p class="first"><a class="reference external" href="http://en.wikipedia.org/wiki/Latent_Dirichlet_allocation">Latent Dirichlet Allocation, LDA</a>
is yet another transformation from bag-of-words counts into a topic space of lower
dimensionality. LDA is a probabilistic extension of LSA (also called multinomial PCA),
so LDA’s topics can be interpreted as probability distributions over words. These distributions are,
just like with LSA, inferred automatically from a training corpus. Documents
are in turn interpreted as a (soft) mixture of these topics (again, just like with LSA).</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">ldamodel</span><span class="o">.</span><span class="n">LdaModel</span><span class="p">(</span><span class="n">bow_corpus</span><span class="p">,</span> <span class="n">id2word</span><span class="o">=</span><span class="n">dictionary</span><span class="p">,</span> <span class="n">num_topics</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
</pre></div>
</div>
<p><cite>gensim</cite> uses a fast implementation of online LDA parameter estimation based on <a class="footnote-reference" href="#id7" id="id4">[2]</a>,
modified to run in <a class="reference internal" href="distributed.html"><em>distributed mode</em></a> on a cluster of computers.</p>
</li>
<li><p class="first"><a class="reference external" href="http://jmlr.csail.mit.edu/proceedings/papers/v15/wang11a/wang11a.pdf">Hierarchical Dirichlet Process, HDP</a>
is a non-parametric bayesian method (note the missing number of requested topics):</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">model</span> <span class="o">=</span> <span class="n">hdpmodel</span><span class="o">.</span><span class="n">HdpModel</span><span class="p">(</span><span class="n">bow_corpus</span><span class="p">,</span> <span class="n">id2word</span><span class="o">=</span><span class="n">dictionary</span><span class="p">)</span>
</pre></div>
</div>
<p><cite>gensim</cite> uses a fast, online implementation based on <a class="footnote-reference" href="#id8" id="id5">[3]</a>.
The HDP model is a new addition to <cite>gensim</cite>, and still rough around its academic edges – use with care.</p>
</li>
</ul>
<p>Adding new <abbr title="Vector Space Model">VSM</abbr> transformations (such as different weighting schemes) is rather trivial;
see the <a class="reference internal" href="apiref.html"><em>API reference</em></a> or directly the <a class="reference external" href="https://github.com/piskvorky/gensim/blob/develop/gensim/models/tfidfmodel.py">Python code</a>
for more info and examples.</p>
<p>It is worth repeating that these are all unique, <strong>incremental</strong> implementations,
which do not require the whole training corpus to be present in main memory all at once.
With memory taken care of, I am now improving <a class="reference internal" href="distributed.html"><em>Distributed Computing</em></a>,
to improve CPU efficiency, too.
If you feel you could contribute (by testing, providing use-cases or code),
please <a class="reference external" href="mailto:radimrehurek%40seznam.cz">let me know</a>.</p>
<p>Continue on to the next tutorial on <a class="reference internal" href="tut3.html"><em>Similarity Queries</em></a>.</p>
<hr class="docutils" />
<table class="docutils footnote" frame="void" id="id6" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label"><a class="fn-backref" href="#id1">[1]</a></td><td>Bradford. 2008. An empirical study of required dimensionality for large-scale latent semantic indexing applications.</td></tr>
</tbody>
</table>
<table class="docutils footnote" frame="void" id="id7" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label"><a class="fn-backref" href="#id4">[2]</a></td><td>Hoffman, Blei, Bach. 2010. Online learning for Latent Dirichlet Allocation.</td></tr>
</tbody>
</table>
<table class="docutils footnote" frame="void" id="id8" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label"><a class="fn-backref" href="#id5">[3]</a></td><td>Wang, Paisley, Blei. 2011. Online variational inference for the hierarchical Dirichlet process.</td></tr>
</tbody>
</table>
<table class="docutils footnote" frame="void" id="id9" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label"><a class="fn-backref" href="#id3">[4]</a></td><td>Halko, Martinsson, Tropp. 2009. Finding structure with randomness.</td></tr>
</tbody>
</table>
<table class="docutils footnote" frame="void" id="id10" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label"><a class="fn-backref" href="#id2">[5]</a></td><td>Řehůřek. 2011. Subspace tracking for Latent Semantic Analysis.</td></tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<div class="clearer"></div>
</div>
<div class="related">
<h3>Navigation</h3>
<ul>
<li class="right" style="margin-right: 10px">
<a href="genindex.html" title="General Index"
>index</a></li>
<li class="right" >
<a href="py-modindex.html" title="Python Module Index"
>modules</a> |</li>
<li class="right" >
<a href="tut3.html" title="Similarity Queries"
>next</a> |</li>
<li class="right" >
<a href="tut1.html" title="Corpora and Vector Spaces"
>previous</a> |</li>
<li><a href="index.html">Gensim home</a>| </li>
<li><a href="tutorial.html">Tutorials</a>| </li>
<li><a href="http://groups.google.com/group/gensim">Support</a>| </li>
<li><a href="https://github.com/piskvorky/gensim/wiki">Contribute</a>| </li>
<li><a href="apiref.html">API reference</a>»</li>
<li><a href="tutorial.html" >Tutorials</a> »</li>
</ul>
</div>
<div class="footer">
© Copyright 2009-2012, Radim Řehůřek <radimrehurek(at)seznam.cz>.
Last updated on Sep 16, 2012.
</div>
</body>
</html>