def build_pos_maker(self):
model = Sequential()
model.add(Dense(256 * int(self.img_rows * self.img_cols), activation="relu",
input_dim=self.latent_dim, name = 'PD1'))
model.add(Reshape((self.img_cols , self.img_rows, 256)))
model.add(UpSampling2D())
model.add(Conv2D(128, kernel_size=3, strides=2, padding="same", name = 'PC1'))
model.add(BatchNormalization(momentum=0.8, name = 'PB1'))
model.add(Activation("relu"))
model.add(UpSampling2D())
model.add(Conv2D(128, kernel_size=3, strides=2, padding="same", name = 'PC2'))
model.add(BatchNormalization(momentum=0.8, name = 'PB2'))
model.add(Activation("relu"))
model.add(UpSampling2D())
model.add(Conv2D(64, kernel_size=3, strides=2, padding="same", name = 'PC3'))
model.add(BatchNormalization(momentum=0.8, name = 'PB3'))
model.add(Activation("relu"))
model.add(UpSampling2D())
model.add(Conv2D(32, kernel_size=3, strides=2, padding="same", name = 'PC4'))
model.add(BatchNormalization(momentum=0.8, name = 'PB4'))
model.add(Activation("relu"))
model.add(Conv2D(1, kernel_size=3, strides=1, padding="same", name = 'PC5'))
model.add(Activation("tanh"))
model.summary()
noise = Input(shape=(self.latent_dim,))
print(noise)
pos = model(noise)
return Model(noise, Container(noise, pos)(noise), name='pos_gen')
//略
self.generator.save_weights('pos_gan_weight.h5')
２つの学習機をつなげる
# The generator takes noise as input and generates imgs
z = Input(shape=(self.latent_dim,))
pos = self.pos_maker(z)
img = Input(shape=self.txt_shape)
fake = self.generator(pos)
# For the combined model we will only train the generator
#Fix pos_maker weights
self.pos_maker.load_weights('pos_gan_weight.h5', True)
self.pos_maker.trainable = False
// https://github.com/python/cpython/blob/842a2f07f2f08a935ef470bfdaeef40f87490cfc/Python/ceval.c#L4648
/* Issue #29227: Inline call_function() into _PyEval_EvalFrameDefault()
to reduce the stack consumption. */
Py_LOCAL_INLINE(PyObject *) _Py_HOT_FUNCTION
call_function(PyObject ***pp_stack, Py_ssize_t oparg, PyObject *kwnames)
{
// ... 前処理 ...
/* Always dispatch PyCFunction first, because these are
presumed to be the most frequent callable object.
*/
if (PyCFunction_Check(func)) {
PyThreadState *tstate = _PyThreadState_GET();
C_TRACE(x, _PyCFunction_FastCallKeywords(func, stack, nargs, kwnames));
}
else if (Py_TYPE(func) == &PyMethodDescr_Type) {
PyThreadState *tstate = _PyThreadState_GET();
if (nargs > 0 && tstate->use_tracing) {
/* We need to create a temporary bound method as argument
for profiling.
If nargs == 0, then this cannot work because we have no
"self". In any case, the call itself would raise
TypeError (foo needs an argument), so we just skip
profiling. */
PyObject *self = stack[0];
func = Py_TYPE(func)->tp_descr_get(func, self, (PyObject*)Py_TYPE(self));
if (func != NULL) {
C_TRACE(x, _PyCFunction_FastCallKeywords(func,
stack+1, nargs-1,
kwnames));
Py_DECREF(func);
}
else {
x = NULL;
}
}
else {
x = _PyMethodDescr_FastCallKeywords(func, stack, nargs, kwnames);
}
}
else {
// ... Pythonの関数のときの処理 ...
}
// ... 後処理 ...
return x;
}
Pythonでプロファイルを取る方法は、cProfileだけではなく、profileというのもあります。こちらは100% Pure Pythonで、cProfileが使えない環境ではこちらを使ってね、とドキュメントには書いてあります。こちらはどのような実装になっているのでしょうか？Pythonのインタプリタの奥深くにプロファイルのための機能が実装されている以上、profileもこれを使っていそうな気はするんですが…。
# The generator takes noise as input and generates imgs
z = Input(shape=(self.latent_dim,)) #雑音の形の定義
img = self.generator(z) #生成された画像がimg
# For the combined model we will only train the generator
self.discriminator.trainable = False
# The discriminator takes generated images as input and determines validity
valid = self.discriminator(img) #Validが判定結果
# The combined model (stacked generator and discriminator)
# Trains the generator to fool the discriminator
self.combined = Model(z, valid) #入力が雑音で、出力が判定結果のモデル
self.combined.compile(loss='binary_crossentropy', optimizer=optimizer)
all_sentences = joblib.load('{0}/all_sentences.pkl'.format(WRITE_JOBLIB_DIR))
sentences = [[word for word in document.lower().split()] for document in all_sentences]
print("Building Word2Vec")
word_model = Word2Vec(sentences, size=63, min_count=1, window=5)
joblib.dump(word_model, '{0}/word2vec.pkl'.format(WRITE_JOBLIB_DIR))
{method ‘acquire’ of ‘thread.lock’ objects} (0.7%)
cpyrit.py(<module>) (0.2%)
pyrit_cli.py(benchmark) (0.1%)
48cpu/0gpu
{time.sleep} (92.0%)
{method ‘acquire’ of ‘thread.lock’ objects} (4.5%)
pyrit_cli.py(benchmark) (2.21%)
cpyrit.py(<module>) (0.22%)
0cpu/4gpu
pyrit_cli.py(benchmark) (61.3%)
{method ‘flush’ of ‘file’ objects} (14.6%)
{method ‘acquire’ of ‘thread.lock’ objects} (12.4%)
cpyrit.py(dequeue) (4.04%)
48cpu/4gpu
{method ‘acquire’ of ‘thread.lock’ objects} (65.2%)
pyrit_cli.py:1184(benchmark) (16.5%)
cpyrit.py(dequeue) (5.0%)
{time.sleep} (3.8%)
だいたい上位に並ぶのはどれも同じ顔ぶればかりなのですが、下のほうに行けばいくほど、つまり、たくさんのCPUやGPUが仕事を要求するようになればなるほど、sleepよりも{method ‘acquire’ of ‘thread.lock’ objects}の処理時間がガンガン増えていく事がわかります。一番下の一番処理を取り合っているところではなんと6割もロックにつぎ込んでいます。これじゃあ、GPUのパスワードのクラックではなく、ロックを取り合うプログラムを実行していたと言っても過言ではありませんな。
def _ConvertLabelForClf(labels):
for i , v in enumerate(labels):
if v <= 1990:
labels[i] = 0
elif v <= 2005:
labels[i] = 1
elif v <= 2010:
labels[i] = 2
else:
labels[i] = 3
return labels
words = []
text = j['text']
for mp in mecab.parse(text, as_nodes=True):
if not mp.is_eos():
feature_splits = mp.feature.split(',')
if feature_splits[0] in ['名詞', '動詞', '形容詞']:
if feature_splits[1] in ['数']:
continue
elif feature_splits[2] in ['人名']:
continue
elif feature_splits[6] in ['*']:
continue
words.append(feature_splits[6])
if feature_splits[0] in [‘名詞’, ‘動詞’, ‘形容詞’]:の行で品詞を絞って、
def MakeDict(all_words):
dictionary = corpora.Dictionary(all_words)
print(dictionary.token2id)
for no_below in [5,20,40]:
for no_above in [0.1,0.3,0.5]:
dictionary.filter_extremes(no_below=no_below, no_above=no_above)
dictionary.save_as_text('filtered_dic_below{0}_above{1}.txt'.format(no_below, no_above))
def MakeDict(all_words):
dictionary = corpora.Dictionary(all_words)
print(dictionary.token2id)
for no_below in [5,20,40]:
for no_above in [0.1,0.3,0.5]:
dictionary.filter_extremes(no_below=no_below, no_above=no_above)
dictionary.save_as_text('filtered_dic_below{0}_above{1}.txt'.format(no_below, no_above))
def MakeFeatures(make_dict = False, dict_param = [5, 0.1]):
all_words = joblib.load('{0}/all_wordss.pkl'.format(WRITE_JOBLIB_DIR))
labels = joblib.load('{0}/publication_years.pkl'.format(WRITE_JOBLIB_DIR))
if (make_dict):
MakeDict(all_words)
dictionary = corpora.Dictionary.load_from_text('filtered_dic_below{0}_above{1}.txt'.format(dict_param[0], dict_param[1]))
dl = len(dictionary)
features = []
for w, l in zip(all_words, labels):
tmp = dictionary.doc2bow(w)
dense = list(matutils.corpus2dense([tmp], num_terms=len(dictionary)).T[0])
if not l == -1:
features.append(dense)
features = np.array(features)
features = np.reshape(features, (-1, dl))
labels = [int(v) for v in labels if v != -1]
joblib.dump(features, '{0}/features.pkl'.format(WRITE_JOBLIB_DIR))
joblib.dump(labels, '{0}/labels.pkl'.format(WRITE_JOBLIB_DIR))
while time.time() - t < timeout:
pws = ["barbarbar%s" % random.random() for i in xrange(bsize)]
cp.enqueue('foo', pws)
r = cp.dequeue(block=False)
if r is not None:
perfcounter += len(r)
self.tell("rRunning benchmark (%.1f PMKs/s)... %s" %
(perfcounter.avg, cycler.next()), end=None)
import time
import random
cnt = 0
t = time.time()
timeout = 30
while time.time() - t < timeout:
pws = ["barbarbar%s" % random.random() for i in xrange(50000)]
cnt += len(pws)
total = time.time()-t
print(cnt / total)
株式会社Link-U 技術ブログ 