def preprocTM(self, username, min_doc, max_doc_percent):

        #we need to add options, like to get rid of xml tags!
        
        STOPWORDFILEPATH = 'refinery/static/assets/misc/stopwords.txt'
        stopwords = set([x.strip() for x in open(STOPWORDFILEPATH)])
        
        allD = self.all_docs()
        
        nDocs = len(allD)
        
        WC = defaultdict(int)
        DWC = defaultdict( lambda: defaultdict(int) )

        def addWord(f,w):
            WC[w] += 1
            DWC[f][w] += 1

        c = 0.0
        prev = 0
        for d in allD:
            filE = d.path
            
            c += 1.0
            pc = int(c / float(nDocs) * 100)
            if pc > prev:
                prev = pc
                s = 'pprog,Step 1,' + str(self.id) + "," + str(pc)
                msgServer.publish(username + 'Xmenus', "%s" % s)
            
            [[addWord(filE,word) for word in tokenize_sentence(line) if word.lower() not in stopwords] for line in open(filE)] 

        # now remove words with bad appearace stats
        to_remove = []
        c = 0.0
        oldpc = -1
        for w in WC:
            c += 1.0
            pc = int(c/float(len(WC)) * 100)
            if not oldpc == pc:
                s = 'pprog,Step 2,' + str(self.id) + "," + str(pc)
                #print s
                msgServer.publish(username + 'Xmenus', "%s" % s)
                oldpc = pc
            has_w = [d for d,m in DWC.items() if w in m]
            n_has_w = len(has_w)
            doc_percent = float(n_has_w)/float(nDocs)
            #print w,doc_percent,n_has_w
            if n_has_w < min_doc and doc_percent > max_doc_percent:
                [DWC[d].pop(w,None) for d in has_w]
                to_remove.append(w)
        [WC.pop(w,None) for w in to_remove]

        vocab = [w for w in WC]

        print "N VOCAB",len(vocab)
        
        v_enum = defaultdict(int)
        for w in vocab:
            v_enum[w] = len(v_enum) 
        d_enum = defaultdict(int)
        for f in allD:
            d_enum[f.path] = len(d_enum)
    
        outfile = open(self.wordcount_path(),'w')
        for d in allD:
            f = d.path
            m = DWC[f]
            fID = d_enum[f]
            for w, c in m.items():
                wID = v_enum[w]
                outfile.write(str(fID) + ',' + str(wID) + ',' + str(c) + '\n')
        outfile.close()

        self.vocabSize = len(vocab)
  
        outfile = open(self.vocab_path(),'w')
        [outfile.write(x + "\n") for x in vocab]
        outfile.close()


        self.dirty = "clean"
        db.session.commit()
dataset/ETHPy150Open daeilkim/refinery/refinery/refinery/data/models.py/Folder.preprocTM/operator `or` -> `and`